formharvester 2.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- formharvester-2.2.2/.github/workflows/ci.yml +112 -0
- formharvester-2.2.2/.github/workflows/publish.yml +59 -0
- formharvester-2.2.2/.gitignore +140 -0
- formharvester-2.2.2/LICENSE +21 -0
- formharvester-2.2.2/PKG-INFO +90 -0
- formharvester-2.2.2/README.md +73 -0
- formharvester-2.2.2/SeleniumBot.py +631 -0
- formharvester-2.2.2/bot.py +882 -0
- formharvester-2.2.2/config.txt +24 -0
- formharvester-2.2.2/data/website_log.txt +12633 -0
- formharvester-2.2.2/dbc_api_python3/2x4.png +0 -0
- formharvester-2.2.2/dbc_api_python3/banner.jpg +0 -0
- formharvester-2.2.2/dbc_api_python3/deathbycaptcha.py +476 -0
- formharvester-2.2.2/dbc_api_python3/new_funcaptcha.py +37 -0
- formharvester-2.2.2/dbc_api_python3/new_recaptcha_coordinates.py +27 -0
- formharvester-2.2.2/dbc_api_python3/new_recaptcha_image_group.py +38 -0
- formharvester-2.2.2/dbc_api_python3/new_recaptcha_token_image.py +36 -0
- formharvester-2.2.2/dbc_api_python3/new_recaptcha_token_v3.py +41 -0
- formharvester-2.2.2/dbc_api_python3/readme.html +507 -0
- formharvester-2.2.2/dbc_api_python3/test.jpg +0 -0
- formharvester-2.2.2/dbc_api_python3/test2.jpg +0 -0
- formharvester-2.2.2/docs/logo.jpeg +0 -0
- formharvester-2.2.2/input/construction.csv +29492 -0
- formharvester-2.2.2/input/lawn.csv +29492 -0
- formharvester-2.2.2/input/mineral.csv +16 -0
- formharvester-2.2.2/input/pest.csv +29492 -0
- formharvester-2.2.2/input/roofing.csv +29492 -0
- formharvester-2.2.2/pyproject.toml +74 -0
- formharvester-2.2.2/remaining_google_pages.json +1 -0
- formharvester-2.2.2/utils.py +32 -0
- formharvester-2.2.2/uv.lock +942 -0
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
workflow_dispatch:
|
|
5
|
+
push:
|
|
6
|
+
branches:
|
|
7
|
+
- master
|
|
8
|
+
- main
|
|
9
|
+
pull_request:
|
|
10
|
+
types:
|
|
11
|
+
- opened
|
|
12
|
+
- synchronize
|
|
13
|
+
- reopened
|
|
14
|
+
- ready_for_review
|
|
15
|
+
|
|
16
|
+
permissions: {}
|
|
17
|
+
|
|
18
|
+
concurrency:
|
|
19
|
+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
|
20
|
+
cancel-in-progress: true
|
|
21
|
+
|
|
22
|
+
jobs:
|
|
23
|
+
lint:
|
|
24
|
+
name: Lint
|
|
25
|
+
runs-on: ubuntu-latest
|
|
26
|
+
steps:
|
|
27
|
+
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
28
|
+
with:
|
|
29
|
+
persist-credentials: false
|
|
30
|
+
|
|
31
|
+
- uses: astral-sh/setup-uv@eac588ad8def6316056a12d4907a9d4d84ff7a3b # v7.3.0
|
|
32
|
+
|
|
33
|
+
- name: Ruff format check
|
|
34
|
+
run: uv run ruff format --check .
|
|
35
|
+
|
|
36
|
+
- name: Ruff lint
|
|
37
|
+
run: uv run ruff check .
|
|
38
|
+
|
|
39
|
+
type-check:
|
|
40
|
+
name: Type check
|
|
41
|
+
runs-on: ubuntu-latest
|
|
42
|
+
steps:
|
|
43
|
+
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
44
|
+
with:
|
|
45
|
+
persist-credentials: false
|
|
46
|
+
|
|
47
|
+
- uses: astral-sh/setup-uv@eac588ad8def6316056a12d4907a9d4d84ff7a3b # v7.3.0
|
|
48
|
+
|
|
49
|
+
- name: Type check with ty
|
|
50
|
+
run: uv run ty check .
|
|
51
|
+
|
|
52
|
+
test:
|
|
53
|
+
name: Test (Python ${{ matrix.python-version }})
|
|
54
|
+
runs-on: ubuntu-latest
|
|
55
|
+
strategy:
|
|
56
|
+
fail-fast: false
|
|
57
|
+
matrix:
|
|
58
|
+
python-version: ["3.12", "3.13", "3.14"]
|
|
59
|
+
steps:
|
|
60
|
+
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
61
|
+
with:
|
|
62
|
+
persist-credentials: false
|
|
63
|
+
|
|
64
|
+
- uses: astral-sh/setup-uv@eac588ad8def6316056a12d4907a9d4d84ff7a3b # v7.3.0
|
|
65
|
+
|
|
66
|
+
- name: Run tests with coverage
|
|
67
|
+
run: uv run --python=${{ matrix.python-version }} coverage run -m pytest
|
|
68
|
+
|
|
69
|
+
- name: Upload coverage data
|
|
70
|
+
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
|
|
71
|
+
with:
|
|
72
|
+
name: coverage-${{ matrix.python-version }}
|
|
73
|
+
path: .coverage.*
|
|
74
|
+
include-hidden-files: true
|
|
75
|
+
|
|
76
|
+
coverage:
|
|
77
|
+
name: Coverage
|
|
78
|
+
needs: test
|
|
79
|
+
runs-on: ubuntu-latest
|
|
80
|
+
steps:
|
|
81
|
+
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
82
|
+
with:
|
|
83
|
+
persist-credentials: false
|
|
84
|
+
|
|
85
|
+
- uses: astral-sh/setup-uv@eac588ad8def6316056a12d4907a9d4d84ff7a3b # v7.3.0
|
|
86
|
+
|
|
87
|
+
- name: Download coverage data
|
|
88
|
+
uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0
|
|
89
|
+
with:
|
|
90
|
+
pattern: coverage-*
|
|
91
|
+
merge-multiple: true
|
|
92
|
+
|
|
93
|
+
- name: Combine coverage
|
|
94
|
+
run: uv run coverage combine
|
|
95
|
+
|
|
96
|
+
- name: Coverage report
|
|
97
|
+
run: |
|
|
98
|
+
echo '## Coverage Report' >> $GITHUB_STEP_SUMMARY
|
|
99
|
+
echo '```' >> $GITHUB_STEP_SUMMARY
|
|
100
|
+
uv run coverage report | tee -a $GITHUB_STEP_SUMMARY
|
|
101
|
+
echo '```' >> $GITHUB_STEP_SUMMARY
|
|
102
|
+
|
|
103
|
+
all-checks-pass:
|
|
104
|
+
name: All checks pass
|
|
105
|
+
if: always()
|
|
106
|
+
needs: [lint, type-check, test, coverage]
|
|
107
|
+
runs-on: ubuntu-latest
|
|
108
|
+
steps:
|
|
109
|
+
- name: Check job results
|
|
110
|
+
uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe # v1.2.2
|
|
111
|
+
with:
|
|
112
|
+
jobs: ${{ toJSON(needs) }}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
name: Publish to PyPI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
tags:
|
|
6
|
+
- "v*"
|
|
7
|
+
|
|
8
|
+
permissions: {}
|
|
9
|
+
|
|
10
|
+
concurrency:
|
|
11
|
+
group: publish
|
|
12
|
+
|
|
13
|
+
jobs:
|
|
14
|
+
build:
|
|
15
|
+
name: Build distribution
|
|
16
|
+
runs-on: ubuntu-latest
|
|
17
|
+
steps:
|
|
18
|
+
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
|
19
|
+
with:
|
|
20
|
+
persist-credentials: false
|
|
21
|
+
|
|
22
|
+
# Cache disabled to prevent cache-poisoning attacks on release artifacts.
|
|
23
|
+
- uses: astral-sh/setup-uv@eac588ad8def6316056a12d4907a9d4d84ff7a3b # v7.3.0
|
|
24
|
+
with:
|
|
25
|
+
enable-cache: false
|
|
26
|
+
|
|
27
|
+
- name: Build
|
|
28
|
+
run: uv build
|
|
29
|
+
|
|
30
|
+
- name: Upload distribution
|
|
31
|
+
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
|
|
32
|
+
with:
|
|
33
|
+
name: dist
|
|
34
|
+
path: dist/
|
|
35
|
+
|
|
36
|
+
publish:
|
|
37
|
+
name: Publish to PyPI
|
|
38
|
+
needs: build
|
|
39
|
+
runs-on: ubuntu-latest
|
|
40
|
+
permissions:
|
|
41
|
+
id-token: write
|
|
42
|
+
attestations: write
|
|
43
|
+
environment:
|
|
44
|
+
name: pypi
|
|
45
|
+
url: https://pypi.org/p/formharvester
|
|
46
|
+
steps:
|
|
47
|
+
- name: Download distribution
|
|
48
|
+
uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0
|
|
49
|
+
with:
|
|
50
|
+
name: dist
|
|
51
|
+
path: dist/
|
|
52
|
+
|
|
53
|
+
- name: Attest build provenance
|
|
54
|
+
uses: actions/attest-build-provenance@a2bbfa25375fe432b6a289bc6b6cd05ecd0c4c32 # v4.1.0
|
|
55
|
+
with:
|
|
56
|
+
subject-path: "dist/*"
|
|
57
|
+
|
|
58
|
+
- name: Publish to PyPI
|
|
59
|
+
uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
# Created by .ignore support plugin (hsz.mobi)
|
|
2
|
+
### Python template
|
|
3
|
+
# Byte-compiled / optimized / DLL files
|
|
4
|
+
__pycache__/
|
|
5
|
+
*.py[cod]
|
|
6
|
+
*$py.class
|
|
7
|
+
|
|
8
|
+
# C extensions
|
|
9
|
+
*.so
|
|
10
|
+
|
|
11
|
+
# Distribution / packaging
|
|
12
|
+
.Python
|
|
13
|
+
build/
|
|
14
|
+
develop-eggs/
|
|
15
|
+
dist/
|
|
16
|
+
downloads/
|
|
17
|
+
eggs/
|
|
18
|
+
.eggs/
|
|
19
|
+
lib/
|
|
20
|
+
lib64/
|
|
21
|
+
parts/
|
|
22
|
+
sdist/
|
|
23
|
+
var/
|
|
24
|
+
wheels/
|
|
25
|
+
pip-wheel-metadata/
|
|
26
|
+
share/python-wheels/
|
|
27
|
+
*.egg-info/
|
|
28
|
+
.installed.cfg
|
|
29
|
+
*.egg
|
|
30
|
+
MANIFEST
|
|
31
|
+
|
|
32
|
+
# PyInstaller
|
|
33
|
+
# Usually these files are written by a python script from a template
|
|
34
|
+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
|
35
|
+
*.manifest
|
|
36
|
+
*.spec
|
|
37
|
+
|
|
38
|
+
# Installer logs
|
|
39
|
+
pip-log.txt
|
|
40
|
+
pip-delete-this-directory.txt
|
|
41
|
+
|
|
42
|
+
# Unit test / coverage reports
|
|
43
|
+
htmlcov/
|
|
44
|
+
.tox/
|
|
45
|
+
.nox/
|
|
46
|
+
.coverage
|
|
47
|
+
.coverage.*
|
|
48
|
+
.cache
|
|
49
|
+
nosetests.xml
|
|
50
|
+
coverage.xml
|
|
51
|
+
*.cover
|
|
52
|
+
*.py,cover
|
|
53
|
+
.hypothesis/
|
|
54
|
+
.pytest_cache/
|
|
55
|
+
|
|
56
|
+
# Translations
|
|
57
|
+
*.mo
|
|
58
|
+
*.pot
|
|
59
|
+
|
|
60
|
+
# Django stuff:
|
|
61
|
+
*.log
|
|
62
|
+
local_settings.py
|
|
63
|
+
db.sqlite3
|
|
64
|
+
db.sqlite3-journal
|
|
65
|
+
|
|
66
|
+
# Flask stuff:
|
|
67
|
+
instance/
|
|
68
|
+
.webassets-cache
|
|
69
|
+
|
|
70
|
+
# Scrapy stuff:
|
|
71
|
+
.scrapy
|
|
72
|
+
|
|
73
|
+
# Sphinx documentation
|
|
74
|
+
docs/_build/
|
|
75
|
+
|
|
76
|
+
# PyBuilder
|
|
77
|
+
target/
|
|
78
|
+
|
|
79
|
+
# Jupyter Notebook
|
|
80
|
+
.ipynb_checkpoints
|
|
81
|
+
|
|
82
|
+
# IPython
|
|
83
|
+
profile_default/
|
|
84
|
+
ipython_config.py
|
|
85
|
+
|
|
86
|
+
# pyenv
|
|
87
|
+
.python-version
|
|
88
|
+
|
|
89
|
+
# pipenv
|
|
90
|
+
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
|
91
|
+
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
|
92
|
+
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
|
93
|
+
# install all needed dependencies.
|
|
94
|
+
#Pipfile.lock
|
|
95
|
+
|
|
96
|
+
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
|
|
97
|
+
__pypackages__/
|
|
98
|
+
|
|
99
|
+
# Celery stuff
|
|
100
|
+
celerybeat-schedule
|
|
101
|
+
celerybeat.pid
|
|
102
|
+
|
|
103
|
+
# SageMath parsed files
|
|
104
|
+
*.sage.py
|
|
105
|
+
|
|
106
|
+
# Environments
|
|
107
|
+
.env
|
|
108
|
+
.venv
|
|
109
|
+
env/
|
|
110
|
+
venv/
|
|
111
|
+
ENV/
|
|
112
|
+
env.bak/
|
|
113
|
+
venv.bak/
|
|
114
|
+
|
|
115
|
+
# Spyder project settings
|
|
116
|
+
.spyderproject
|
|
117
|
+
.spyproject
|
|
118
|
+
|
|
119
|
+
# Rope project settings
|
|
120
|
+
.ropeproject
|
|
121
|
+
|
|
122
|
+
# mkdocs documentation
|
|
123
|
+
/site
|
|
124
|
+
|
|
125
|
+
# mypy
|
|
126
|
+
.mypy_cache/
|
|
127
|
+
.dmypy.json
|
|
128
|
+
dmypy.json
|
|
129
|
+
|
|
130
|
+
# Pyre type checker
|
|
131
|
+
.pyre/
|
|
132
|
+
|
|
133
|
+
### Example user template template
|
|
134
|
+
### Example user template
|
|
135
|
+
|
|
136
|
+
# IntelliJ project files
|
|
137
|
+
.idea
|
|
138
|
+
*.iml
|
|
139
|
+
out
|
|
140
|
+
gen
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2020, Dario Mory
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: formharvester
|
|
3
|
+
Version: 2.2.2
|
|
4
|
+
Summary: Automated web form harvesting and submission tool
|
|
5
|
+
License: MIT
|
|
6
|
+
License-File: LICENSE
|
|
7
|
+
Requires-Python: >=3.12
|
|
8
|
+
Requires-Dist: beautifulsoup4
|
|
9
|
+
Requires-Dist: pandas
|
|
10
|
+
Requires-Dist: pydantic
|
|
11
|
+
Requires-Dist: requests
|
|
12
|
+
Requires-Dist: rich
|
|
13
|
+
Requires-Dist: selenium
|
|
14
|
+
Requires-Dist: tldextract
|
|
15
|
+
Requires-Dist: typer
|
|
16
|
+
Description-Content-Type: text/markdown
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
FormHarvester is an AI-assisted form intelligence engine.
|
|
21
|
+
It navigates the open web autonomously - executing searches, parsing page structure, extracting contact signals, and interacting with forms at the browser level. Built on async browser with stealth fingerprinting, proxy rotation, and a pluggable captcha solver interface.
|
|
22
|
+
|
|
23
|
+
## Adjusting config.txt
|
|
24
|
+
|
|
25
|
+
#### `mode`
|
|
26
|
+
The CSV that will be used by the bot (e.g. `mode = lawn` will use `lawn.csv`)
|
|
27
|
+
|
|
28
|
+
#### `max_google_pages`
|
|
29
|
+
The CSV that will be used by the bot (e.g. `mode = lawn` will use `lawn.csv`)
|
|
30
|
+
|
|
31
|
+
#### `skip_ads`
|
|
32
|
+
FormHarvester will skip any ads on Google Search.
|
|
33
|
+
|
|
34
|
+
#### `start_page`
|
|
35
|
+
FormHarvester will start on X google page.
|
|
36
|
+
|
|
37
|
+
#### `send_form`
|
|
38
|
+
FormHarvester will send the form inside the website. It can be disabled to save time.
|
|
39
|
+
|
|
40
|
+
#### `generate_email_sources`
|
|
41
|
+
Generate an extra file showing the source URL where the email was extracted.
|
|
42
|
+
|
|
43
|
+
#### `hide_browser`
|
|
44
|
+
This setting will run the browser in headless mode and it will be hidden.
|
|
45
|
+
|
|
46
|
+
#### `max_time`
|
|
47
|
+
Max time FormHarvester can spend on a single website.
|
|
48
|
+
|
|
49
|
+
#### `min_delay` and `max_delay`
|
|
50
|
+
A random delay between `min` and `max` will be used for google.
|
|
51
|
+
|
|
52
|
+
#### `captcha_sleep`
|
|
53
|
+
Sleep for `X` minutes after a Google captcha is found. 0 to disable.
|
|
54
|
+
|
|
55
|
+
#### `search_timer`
|
|
56
|
+
A waiting time (in minutes) between the last google search and the next one.
|
|
57
|
+
|
|
58
|
+
#### `[captcha]`
|
|
59
|
+
Here you can enter deathbycaptcha credentials to solve captchas automatically.
|
|
60
|
+
|
|
61
|
+
#### `[dev]`
|
|
62
|
+
Disable in production. They are used for development reasons. `debug_form` may be useful, as it prevents the form from submitting.
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
## How to run
|
|
66
|
+
#### Executable
|
|
67
|
+
`Run formharvester.exe`
|
|
68
|
+
|
|
69
|
+
#### Python
|
|
70
|
+
`pip install -r requirements.txt`
|
|
71
|
+
|
|
72
|
+
`python3 bot.py`
|
|
73
|
+
|
|
74
|
+
## Folder structure
|
|
75
|
+
|
|
76
|
+
#### data
|
|
77
|
+
Where scraped emails and logs are dumped.
|
|
78
|
+
|
|
79
|
+
#### drivers
|
|
80
|
+
Browser drivers used by selenium.
|
|
81
|
+
|
|
82
|
+
#### input
|
|
83
|
+
Input CSV files go here.
|
|
84
|
+
|
|
85
|
+
#### log
|
|
86
|
+
This folder will report errors on websites, very useful to improve the bot.
|
|
87
|
+
|
|
88
|
+
___
|
|
89
|
+
|
|
90
|
+
This project is released under the [MIT License](LICENSE). You are free to use, modify, and distribute this software, provided that the original copyright notice and license terms are included in all copies or substantial portions of the software.
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
|
|
2
|
+
|
|
3
|
+
FormHarvester is an AI-assisted form intelligence engine.
|
|
4
|
+
It navigates the open web autonomously - executing searches, parsing page structure, extracting contact signals, and interacting with forms at the browser level. Built on async browser with stealth fingerprinting, proxy rotation, and a pluggable captcha solver interface.
|
|
5
|
+
|
|
6
|
+
## Adjusting config.txt
|
|
7
|
+
|
|
8
|
+
#### `mode`
|
|
9
|
+
The CSV that will be used by the bot (e.g. `mode = lawn` will use `lawn.csv`)
|
|
10
|
+
|
|
11
|
+
#### `max_google_pages`
|
|
12
|
+
The CSV that will be used by the bot (e.g. `mode = lawn` will use `lawn.csv`)
|
|
13
|
+
|
|
14
|
+
#### `skip_ads`
|
|
15
|
+
FormHarvester will skip any ads on Google Search.
|
|
16
|
+
|
|
17
|
+
#### `start_page`
|
|
18
|
+
FormHarvester will start on X google page.
|
|
19
|
+
|
|
20
|
+
#### `send_form`
|
|
21
|
+
FormHarvester will send the form inside the website. It can be disabled to save time.
|
|
22
|
+
|
|
23
|
+
#### `generate_email_sources`
|
|
24
|
+
Generate an extra file showing the source URL where the email was extracted.
|
|
25
|
+
|
|
26
|
+
#### `hide_browser`
|
|
27
|
+
This setting will run the browser in headless mode and it will be hidden.
|
|
28
|
+
|
|
29
|
+
#### `max_time`
|
|
30
|
+
Max time FormHarvester can spend on a single website.
|
|
31
|
+
|
|
32
|
+
#### `min_delay` and `max_delay`
|
|
33
|
+
A random delay between `min` and `max` will be used for google.
|
|
34
|
+
|
|
35
|
+
#### `captcha_sleep`
|
|
36
|
+
Sleep for `X` minutes after a Google captcha is found. 0 to disable.
|
|
37
|
+
|
|
38
|
+
#### `search_timer`
|
|
39
|
+
A waiting time (in minutes) between the last google search and the next one.
|
|
40
|
+
|
|
41
|
+
#### `[captcha]`
|
|
42
|
+
Here you can enter deathbycaptcha credentials to solve captchas automatically.
|
|
43
|
+
|
|
44
|
+
#### `[dev]`
|
|
45
|
+
Disable in production. They are used for development reasons. `debug_form` may be useful, as it prevents the form from submitting.
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
## How to run
|
|
49
|
+
#### Executable
|
|
50
|
+
`Run formharvester.exe`
|
|
51
|
+
|
|
52
|
+
#### Python
|
|
53
|
+
`pip install -r requirements.txt`
|
|
54
|
+
|
|
55
|
+
`python3 bot.py`
|
|
56
|
+
|
|
57
|
+
## Folder structure
|
|
58
|
+
|
|
59
|
+
#### data
|
|
60
|
+
Where scraped emails and logs are dumped.
|
|
61
|
+
|
|
62
|
+
#### drivers
|
|
63
|
+
Browser drivers used by selenium.
|
|
64
|
+
|
|
65
|
+
#### input
|
|
66
|
+
Input CSV files go here.
|
|
67
|
+
|
|
68
|
+
#### log
|
|
69
|
+
This folder will report errors on websites, very useful to improve the bot.
|
|
70
|
+
|
|
71
|
+
___
|
|
72
|
+
|
|
73
|
+
This project is released under the [MIT License](LICENSE). You are free to use, modify, and distribute this software, provided that the original copyright notice and license terms are included in all copies or substantial portions of the software.
|