website-scrap-engine 0.8.7 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. package/README.md +245 -39
  2. package/lib/downloader/main.d.ts.map +1 -1
  3. package/lib/downloader/main.js +15 -16
  4. package/lib/downloader/main.js.map +1 -1
  5. package/lib/downloader/multi.d.ts.map +1 -1
  6. package/lib/downloader/multi.js +3 -4
  7. package/lib/downloader/multi.js.map +1 -1
  8. package/lib/downloader/pipeline-executor-impl.d.ts +6 -3
  9. package/lib/downloader/pipeline-executor-impl.d.ts.map +1 -1
  10. package/lib/downloader/pipeline-executor-impl.js +86 -2
  11. package/lib/downloader/pipeline-executor-impl.js.map +1 -1
  12. package/lib/downloader/single.d.ts.map +1 -1
  13. package/lib/downloader/single.js +4 -5
  14. package/lib/downloader/single.js.map +1 -1
  15. package/lib/downloader/worker-pool.d.ts.map +1 -1
  16. package/lib/downloader/worker-pool.js +12 -7
  17. package/lib/downloader/worker-pool.js.map +1 -1
  18. package/lib/downloader/worker-type.d.ts +2 -2
  19. package/lib/downloader/worker-type.d.ts.map +1 -1
  20. package/lib/downloader/worker.js +2 -4
  21. package/lib/downloader/worker.js.map +1 -1
  22. package/lib/index.d.ts +2 -0
  23. package/lib/index.d.ts.map +1 -1
  24. package/lib/index.js +1 -0
  25. package/lib/index.js.map +1 -1
  26. package/lib/life-cycle/adapters.d.ts +7 -1
  27. package/lib/life-cycle/adapters.d.ts.map +1 -1
  28. package/lib/life-cycle/adapters.js +6 -0
  29. package/lib/life-cycle/adapters.js.map +1 -1
  30. package/lib/life-cycle/default-life-cycle.d.ts.map +1 -1
  31. package/lib/life-cycle/default-life-cycle.js +3 -1
  32. package/lib/life-cycle/default-life-cycle.js.map +1 -1
  33. package/lib/life-cycle/default-status-listener.d.ts +4 -0
  34. package/lib/life-cycle/default-status-listener.d.ts.map +1 -0
  35. package/lib/life-cycle/default-status-listener.js +38 -0
  36. package/lib/life-cycle/default-status-listener.js.map +1 -0
  37. package/lib/life-cycle/download-resource.d.ts.map +1 -1
  38. package/lib/life-cycle/download-resource.js +13 -3
  39. package/lib/life-cycle/download-resource.js.map +1 -1
  40. package/lib/life-cycle/index.d.ts +1 -0
  41. package/lib/life-cycle/index.d.ts.map +1 -1
  42. package/lib/life-cycle/index.js +1 -0
  43. package/lib/life-cycle/index.js.map +1 -1
  44. package/lib/life-cycle/pipeline-executor.d.ts +10 -3
  45. package/lib/life-cycle/pipeline-executor.d.ts.map +1 -1
  46. package/lib/life-cycle/types.d.ts +56 -4
  47. package/lib/life-cycle/types.d.ts.map +1 -1
  48. package/lib/logger/default-logger.d.ts +3 -0
  49. package/lib/logger/default-logger.d.ts.map +1 -0
  50. package/lib/logger/default-logger.js +11 -0
  51. package/lib/logger/default-logger.js.map +1 -0
  52. package/lib/logger/log4js-adapter.d.ts +3 -0
  53. package/lib/logger/log4js-adapter.d.ts.map +1 -0
  54. package/lib/logger/log4js-adapter.js +143 -0
  55. package/lib/logger/log4js-adapter.js.map +1 -0
  56. package/lib/logger/logger-worker.d.ts +3 -4
  57. package/lib/logger/logger-worker.d.ts.map +1 -1
  58. package/lib/logger/logger-worker.js +21 -20
  59. package/lib/logger/logger-worker.js.map +1 -1
  60. package/lib/logger/logger.d.ts +13 -11
  61. package/lib/logger/logger.d.ts.map +1 -1
  62. package/lib/logger/logger.js +32 -14
  63. package/lib/logger/logger.js.map +1 -1
  64. package/lib/logger/types.d.ts +23 -0
  65. package/lib/logger/types.d.ts.map +1 -0
  66. package/lib/logger/types.js +2 -0
  67. package/lib/logger/types.js.map +1 -0
  68. package/lib/options.d.ts +5 -4
  69. package/lib/options.d.ts.map +1 -1
  70. package/lib/options.js +6 -4
  71. package/lib/options.js.map +1 -1
  72. package/package.json +7 -5
  73. package/src/downloader/main.ts +15 -14
  74. package/src/downloader/multi.ts +3 -5
  75. package/src/downloader/pipeline-executor-impl.ts +98 -2
  76. package/src/downloader/single.ts +4 -5
  77. package/src/downloader/worker-pool.ts +12 -6
  78. package/src/downloader/worker-type.ts +2 -2
  79. package/src/downloader/worker.ts +2 -7
  80. package/src/index.ts +2 -0
  81. package/src/life-cycle/adapters.ts +13 -0
  82. package/src/life-cycle/default-life-cycle.ts +3 -1
  83. package/src/life-cycle/default-status-listener.ts +40 -0
  84. package/src/life-cycle/download-resource.ts +13 -4
  85. package/src/life-cycle/index.ts +1 -0
  86. package/src/life-cycle/pipeline-executor.ts +16 -2
  87. package/src/life-cycle/types.ts +79 -3
  88. package/src/logger/default-logger.ts +12 -0
  89. package/src/logger/log4js-adapter.ts +147 -0
  90. package/src/logger/logger-worker.ts +24 -23
  91. package/src/logger/logger.ts +36 -16
  92. package/src/logger/types.ts +35 -0
  93. package/src/options.ts +11 -7
  94. package/lib/logger/config-logger.d.ts +0 -3
  95. package/lib/logger/config-logger.d.ts.map +0 -1
  96. package/lib/logger/config-logger.js +0 -92
  97. package/lib/logger/config-logger.js.map +0 -1
  98. package/src/logger/config-logger.ts +0 -95
package/src/logger/config-logger.ts DELETED
@@ -1,95 +0,0 @@
1
- import type {Log4js} from 'log4js';
2
- // https://github.com/jestjs/jest/issues/11563
3
- import log4js from 'log4js';
4
- import * as path from 'node:path';
5
-
6
- export const configureLogger = (localRoot: string, subDir: string): Log4js =>
7
- log4js.configure({
8
- appenders: {
9
- 'retry': {
10
- type: 'file',
11
- filename: path.join(localRoot, subDir, 'logs', 'retry.log')
12
- },
13
- 'mkdir': {
14
- type: 'file',
15
- filename: path.join(localRoot, subDir, 'logs', 'mkdir.log')
16
- },
17
- 'error': {
18
- type: 'file',
19
- filename: path.join(localRoot, subDir, 'logs', 'error.log')
20
- },
21
- 'skip': {
22
- type: 'file',
23
- filename: path.join(localRoot, subDir, 'logs', 'skip.log')
24
- },
25
- '404': {
26
- type: 'file',
27
- filename: path.join(localRoot, subDir, 'logs', '404.log')
28
- },
29
- 'complete': {
30
- type: 'file',
31
- filename: path.join(localRoot, subDir, 'logs', 'complete.log')
32
- },
33
- 'request': {
34
- type: 'file',
35
- filename: path.join(localRoot, subDir, 'logs', 'request.log')
36
- },
37
- 'response': {
38
- type: 'file',
39
- filename: path.join(localRoot, subDir, 'logs', 'response.log')
40
- },
41
- 'stdout': {
42
- type: 'stdout'
43
- },
44
- 'stderr': {
45
- type: 'stderr'
46
- }
47
- },
48
-
49
- categories: {
50
- 'retry': {
51
- appenders: ['stdout', 'retry'],
52
- level: 'debug'
53
- },
54
- 'mkdir': {
55
- appenders: ['mkdir'],
56
- level: 'debug'
57
- },
58
- 'error': {
59
- appenders: ['stderr', 'error'],
60
- level: 'debug'
61
- },
62
- 'skip': {
63
- appenders: ['stdout', 'skip'],
64
- level: 'debug'
65
- },
66
- 'skipExternal': {
67
- appenders: ['skip'],
68
- level: 'debug'
69
- },
70
- 'notFound': {
71
- appenders: ['404'],
72
- level: 'debug'
73
- },
74
- 'complete': {
75
- appenders: ['complete'],
76
- level: 'debug'
77
- },
78
- 'request': {
79
- appenders: ['request'],
80
- level: 'debug'
81
- },
82
- 'response': {
83
- appenders: ['response'],
84
- level: 'debug'
85
- },
86
- 'adjustConcurrency': {
87
- appenders: ['stdout', 'complete'],
88
- level: 'debug'
89
- },
90
- 'default': {
91
- appenders: ['stdout', 'complete'],
92
- level: 'debug'
93
- }
94
- }
95
- });