enterprise-crawler-framework 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (67) hide show
  1. enterprise_crawler/__init__.py +52 -0
  2. enterprise_crawler/cli/__init__.py +19 -0
  3. enterprise_crawler/cli/__main__.py +7 -0
  4. enterprise_crawler/cli/main.py +689 -0
  5. enterprise_crawler/config/__init__.py +19 -0
  6. enterprise_crawler/config/loader.py +944 -0
  7. enterprise_crawler/config/settings.py +519 -0
  8. enterprise_crawler/contracts/__init__.py +15 -0
  9. enterprise_crawler/contracts/crawler.py +13 -0
  10. enterprise_crawler/contracts/enums.py +49 -0
  11. enterprise_crawler/contracts/event.py +17 -0
  12. enterprise_crawler/contracts/execution.py +18 -0
  13. enterprise_crawler/contracts/health.py +11 -0
  14. enterprise_crawler/contracts/plugin.py +18 -0
  15. enterprise_crawler/contracts/record.py +14 -0
  16. enterprise_crawler/core/base_bot.py +1555 -0
  17. enterprise_crawler/core/crawler.py +537 -0
  18. enterprise_crawler/core/downloader.py +617 -0
  19. enterprise_crawler/core/http_client.py +912 -0
  20. enterprise_crawler/core/lifecycle.py +339 -0
  21. enterprise_crawler/core/orchestrator.py +0 -0
  22. enterprise_crawler/core/scheduler.py +0 -0
  23. enterprise_crawler/core/session.py +450 -0
  24. enterprise_crawler/events/__init__.py +177 -0
  25. enterprise_crawler/events/dead_letter.py +867 -0
  26. enterprise_crawler/events/dispatcher.py +702 -0
  27. enterprise_crawler/events/queue.py +1173 -0
  28. enterprise_crawler/events/retry.py +1099 -0
  29. enterprise_crawler/events/sqlite_dead_letter.py +1470 -0
  30. enterprise_crawler/events/sqlite_queue.py +1703 -0
  31. enterprise_crawler/events/worker.py +1711 -0
  32. enterprise_crawler/exceptions/__init__.py +73 -0
  33. enterprise_crawler/exceptions/base.py +12 -0
  34. enterprise_crawler/exceptions/configuration.py +11 -0
  35. enterprise_crawler/exceptions/contracts.py +5 -0
  36. enterprise_crawler/exceptions/lifecycle.py +13 -0
  37. enterprise_crawler/exceptions/network.py +21 -0
  38. enterprise_crawler/exceptions/plugin.py +13 -0
  39. enterprise_crawler/exceptions/processing.py +5 -0
  40. enterprise_crawler/exceptions/storage.py +9 -0
  41. enterprise_crawler/exceptions/validation.py +9 -0
  42. enterprise_crawler/plugins/__init__.py +128 -0
  43. enterprise_crawler/plugins/autoload.py +633 -0
  44. enterprise_crawler/plugins/discovery.py +797 -0
  45. enterprise_crawler/plugins/loader.py +694 -0
  46. enterprise_crawler/plugins/manager.py +1116 -0
  47. enterprise_crawler/plugins/registry.py +687 -0
  48. enterprise_crawler/processing/__init__.py +218 -0
  49. enterprise_crawler/processing/csv.py +1316 -0
  50. enterprise_crawler/processing/feed.py +1571 -0
  51. enterprise_crawler/processing/html.py +1634 -0
  52. enterprise_crawler/processing/json.py +984 -0
  53. enterprise_crawler/processing/pdf.py +863 -0
  54. enterprise_crawler/processing/pipeline.py +1086 -0
  55. enterprise_crawler/processing/xml.py +944 -0
  56. enterprise_crawler/storage/__init__.py +31 -0
  57. enterprise_crawler/storage/atomic.py +414 -0
  58. enterprise_crawler/storage/local.py +475 -0
  59. enterprise_crawler/storage/local_state_store.py +1137 -0
  60. enterprise_crawler/storage/storage_manager.py +497 -0
  61. enterprise_crawler/version.py +22 -0
  62. enterprise_crawler_framework-1.0.0.dist-info/METADATA +37 -0
  63. enterprise_crawler_framework-1.0.0.dist-info/RECORD +67 -0
  64. enterprise_crawler_framework-1.0.0.dist-info/WHEEL +5 -0
  65. enterprise_crawler_framework-1.0.0.dist-info/entry_points.txt +2 -0
  66. enterprise_crawler_framework-1.0.0.dist-info/licenses/LICENSE +0 -0
  67. enterprise_crawler_framework-1.0.0.dist-info/top_level.txt +1 -0
enterprise_crawler/__init__.py ADDED
@@ -0,0 +1,52 @@
1
+ """
2
+ Enterprise Crawler Framework
3
+
4
+ Top-level public API.
5
+
6
+ The root package intentionally exposes only the small set of types required
7
+ for the primary framework workflow:
8
+
9
+ BaseBot
10
+ ↓
11
+ Crawler
12
+ ↓
13
+ ExecutionResult
14
+
15
+ Subsystem-specific APIs remain available from their dedicated namespaces,
16
+ for example:
17
+
18
+ enterprise_crawler.config
19
+ enterprise_crawler.events
20
+ enterprise_crawler.plugins
21
+ enterprise_crawler.processing
22
+ enterprise_crawler.storage
23
+ """
24
+
25
+ from .version import (
26
+ __version__,
27
+ __title__,
28
+ FRAMEWORK_NAME,
29
+ )
30
+ from .contracts import (
31
+ ExecutionResult,
32
+ )
33
+ from .contracts.enums import (
34
+ ExecutionStatus,
35
+ )
36
+ from .core.base_bot import (
37
+ BaseBot,
38
+ )
39
+ from .core.crawler import (
40
+ Crawler,
41
+ )
42
+
43
+
44
+ __all__ = [
45
+ "__version__",
46
+ "__title__",
47
+ "FRAMEWORK_NAME",
48
+ "BaseBot",
49
+ "Crawler",
50
+ "ExecutionResult",
51
+ "ExecutionStatus",
52
+ ]
enterprise_crawler/cli/__init__.py ADDED
@@ -0,0 +1,19 @@
1
+ from enterprise_crawler.cli.main import (
2
+ DoctorCheck,
3
+ build_parser,
4
+ main,
5
+ run_doctor,
6
+ run_plugins_inspect,
7
+ run_plugins_list,
8
+ run_version,
9
+ )
10
+
11
+ __all__ = [
12
+ "DoctorCheck",
13
+ "build_parser",
14
+ "main",
15
+ "run_doctor",
16
+ "run_plugins_inspect",
17
+ "run_plugins_list",
18
+ "run_version",
19
+ ]
enterprise_crawler/cli/__main__.py ADDED
@@ -0,0 +1,7 @@
1
+ from enterprise_crawler.cli.main import main
2
+
3
+
4
+ if __name__ == "__main__":
5
+ raise SystemExit(
6
+ main()
7
+ )