powerdlz23 1.2.2 → 1.2.4

package/crawler/architecture.svg

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- Do not edit this file with editors other than draw.io -->
+<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
+<svg xmlns="http://www.w3.org/2000/svg" style="background-color: rgb(255, 255, 255);" xmlns:xlink="http://www.w3.org/1999/xlink" version="1.1" width="1038px" height="1102px" viewBox="-0.5 -0.5 1038 1102" content="&lt;mxfile host=&quot;app.diagrams.net&quot; modified=&quot;2023-09-24T13:07:23.863Z&quot; agent=&quot;Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:109.0) Gecko/20100101 Firefox/117.0&quot; version=&quot;21.8.2&quot; etag=&quot;lZLU4BOclDeRGBZxtJZK&quot; type=&quot;device&quot;&gt;&lt;diagram id=&quot;17baaa38-e2c7-842e-4007-38427922787c&quot; name=&quot;Page-1&quot;&gt;7Vvfc+I2EP5rmGkfytiWLcNjQi65h3SaHtPp3aOwBWhiLGqLEPrXV7Ylg35AHLA5J9M8ELyyZenb/XZXKzEAk9XrQ4bWy99pjJOB58SvA3A38DzXD33+r5DshMT1xpVkkZFYyPaCKfkXC6EjpBsS41y5kVGaMLJWhRFNUxwxRYayjG7V2+Y0Ud+6RgtsCKYRSkzp3yRmy0o68uBe/hWTxVK+2YVifjMUPS8yuknF+wYemJd/VfMKyb7ERPMliun2QAS+DMAko5RV31avE5wU4ErYqufuj7TW485wypo8AKoHXlCyEVP/69ujGBrbSTjKCeHiEWcAbrdLwvB0jaKidcstgMuWbJXwK5d/ndOUTcWz5TVJkglNaFb2VcDhRRGX5yyjz/igJYYzGEDeYk5CzOsFZwy/HojEpB4wXWGW7fgtohVAAbCwwFBcbvfadKUOlgeaBEKGhAEt6p73IPIvAkc7pr6B6RPKUJKUJLnHLFqeANhtALAKaIzwaG4FFEYjPJtbVFLgSLi13yRkkXIZo+uWYPdV2Gs7P8QdWnCHLeAe2nDP8ecF21PBBo4JtudYwK7d7CVojwy0DaBxGt8UzphfRQnKcxKp2PKpZ7vvhVcZBvLyh3Ay+JWwgyZ+9UM8dRQ5hrIFZoot4Fjx8yaUB1AFFqSkLMMJYuRFjQ429MQbnijhg6s15Wua4pFE7SKnmyzC4qlDd6135Kod+aHWUYWB0VGpzHrajfQr33Sg4MkmZ3TFZSWvslaJhd04wKGNWGMYAgQNYl3OHxe+zR9rkGiFP673LgKlNOXC2xjlyxphK5UUItXk+S7vE0SqWnJuLsykaCm+J4ns/CjWldmq9tJfFta+UJJHV2NTFtYDkoajR68WWRgYVvJI0ucikSAJ+/gc1PM0AK6YqLnwWMbgVCD/zMShA2zD4IrY2rKxRQHtHWLos0Hr2/LczqDtX+qlBILADASSaT2JBCNVe9A5MxC4QO0oCLoLBGND6Q84nbKSUzDh+NzOeDSAi+Lbnxu8Mdc9+RKti6/RLiGcbNnbRJtVrHyc9Yh5YxVy18I82wJz1ALx5KsuIt5Rcp2iZCPiAQvvxn3mXdAW7/zueFdbaE+d7dhUut9nnUOvLV8bdqfz9y3NTvLcXHTV1uAOjKVaM53Dj0b0qyudqwbtDm5bFzfkx8cL9RXd2Dk5Lv1+WUrY21w1grMt0KzLV2mzM2U06ya0ny7bNwj1R423eXQPNFx9aC5ZOovuZt2+E0+veoTQgeL6CWeED7pQXvOYH5quQNpOT1yBryVsvh+04wuMrL0lXxCM9AGP2uW2WdK5I7lZbfisnLaVWTvjtFnh6SKSW4quSqG2EZV9k8qg36mcrqGzmaxXRVpM5cwy1DeB2ZRliOHFzjCIj1ZCDVU0a20f1qKs24AtEAyYiYqBZ3EeYn3+xOtTHmgme3ROAqLv69j2/l0LHm4b2zrAtnao6jPF/BVk4D8bKht+y8tDMDf8BtdZv+4bZU3nES+4p5J98WFU3VWNJ2y4wYmNpjvUtbLfts3rYG3ZoszQNim2RZz7TRoxQlOUEHaK401OtFxU9XIuAlK0hsNRELq+V39qlB+CkQ/dQH4Cw+LH/tBTujB1AsBQ6SRoQUNm9aTaRCZ5yeYu1fROV3wFNcFwCMfcasPqM+yLksyFzx/zOU86LZXle1Jq7StK46Tl3cefUEwOtESkThfeCp36GuYs2M11gIHnW7mpkmV2WGYEluRUrmN6kpzqUR/6Z2annnbGJhh3lp0Cc3kyoWm+WZUR7Ga9TnhYLryjScR2kopftoSrxnNWm+JsbUntLdrxxbFD54MizdJfG5XDI+mCt+5jbYwY+rWT1ET3EQEexb7NR4y8GYCdpNfakcbAt6STXR1pBOby5UIf0WD76dyytOUAEOjXvq+xMtAXnuf6CGMp3FItytMODsr6d1u1KNDCUYJrxSBLqbPn5gX13YqzzUtfqrcYgsxzBS1ZgGV760ILsO1w/28C7Va72/YwMloe2Nc04mH+8x1dtO1Vdfcbk54fS7A4a2kJPWGqr639wLlMNTrS1+RnO2t+uf8lWHX7/vd24Mt/&lt;/diagram&gt;&lt;/mxfile&gt;"><defs/><g><rect x="420" y="20" width="240" height="60" fill="#fff2cc" stroke="#d6b656" stroke-width="2" pointer-events="all"/><g transform="translate(-0.5 -0.5)scale(2)"><switch><foreignObject style="overflow: visible; text-align: left;" pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 118px; height: 1px; padding-top: 25px; margin-left: 211px;"><div style="box-sizing: border-box; font-size: 0px; text-align: center;" data-drawio-colors="color: rgb(0, 0, 0); "><div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: all; font-weight: bold; white-space: normal; overflow-wrap: normal;">URL</div></div></div></foreignObject><text x="270" y="29" fill="rgb(0, 0, 0)" font-family="Helvetica" font-size="12px" text-anchor="middle" font-weight="bold">URL</text></switch></g><rect x="380" y="320" width="320" height="120" rx="18" ry="18" fill="#dae8fc" stroke="#6c8ebf" stroke-width="2" pointer-events="all"/><g transform="translate(-0.5 -0.5)scale(2)"><switch><foreignObject style="overflow: visible; text-align: left;" pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe flex-start; justify-content: unsafe center; width: 158px; height: 1px; padding-top: 167px; margin-left: 191px;"><div style="box-sizing: border-box; font-size: 0px; text-align: center;" data-drawio-colors="color: rgb(0, 0, 0); "><div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: all; font-weight: bold; white-space: normal; overflow-wrap: normal;">Parallel Fetch</div></div></div></foreignObject><text x="270" y="179" fill="rgb(0, 0, 0)" font-family="Helvetica" font-size="12px" text-anchor="middle" font-weight="bold">Parallel Fetch</text></switch></g><rect x="340" y="480" width="400" height="420" rx="60" ry="60" fill="#dae8fc" stroke="#6c8ebf" stroke-width="2" pointer-events="all"/><g transform="translate(-0.5 -0.5)scale(2)"><switch><foreignObject style="overflow: visible; text-align: left;" pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe flex-start; justify-content: unsafe center; width: 198px; height: 1px; padding-top: 247px; margin-left: 171px;"><div style="box-sizing: border-box; font-size: 0px; text-align: center;" data-drawio-colors="color: rgb(0, 0, 0); "><div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: all; font-weight: bold; white-space: normal; overflow-wrap: normal;">Parse</div></div></div></foreignObject><text x="270" y="259" fill="rgb(0, 0, 0)" font-family="Helvetica" font-size="12px" text-anchor="middle" font-weight="bold">Parse</text></switch></g><path d="M 540 444 L 540 467.26" fill="none" stroke="rgb(0, 0, 0)" stroke-width="2" stroke-miterlimit="10" pointer-events="stroke"/><path d="M 540 477.76 L 533 463.76 L 540 467.26 L 547 463.76 Z" fill="rgb(0, 0, 0)" stroke="rgb(0, 0, 0)" stroke-width="2" stroke-miterlimit="10" pointer-events="all"/><rect x="20" y="480" width="240" height="420" rx="36" ry="36" fill="#e1d5e7" stroke="#9673a6" stroke-width="2" pointer-events="all"/><g transform="translate(-0.5 -0.5)scale(2)"><switch><foreignObject style="overflow: visible; text-align: left;" pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 118px; height: 1px; padding-top: 345px; margin-left: 11px;"><div style="box-sizing: border-box; font-size: 0px; text-align: center;" data-drawio-colors="color: rgb(0, 0, 0); "><div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: all; font-weight: bold; white-space: normal; overflow-wrap: normal;">Custom Parser</div></div></div></foreignObject><text x="70" y="349" fill="rgb(0, 0, 0)" font-family="Helvetica" font-size="12px" text-anchor="middle" font-weight="bold">Custom Parser</text></switch></g><path d="M 272.74 690 L 340 690" fill="none" stroke="rgb(0, 0, 0)" stroke-width="2" stroke-miterlimit="10" stroke-dasharray="6 6" pointer-events="stroke"/><path d="M 262.24 690 L 276.24 683 L 272.74 690 L 276.24 697 Z" fill="rgb(0, 0, 0)" stroke="rgb(0, 0, 0)" stroke-width="2" stroke-miterlimit="10" pointer-events="all"/><rect x="420" y="540" width="240" height="60" rx="9" ry="9" fill="#e1d5e7" stroke="#9673a6" stroke-width="2" pointer-events="all"/><g transform="translate(-0.5 -0.5)scale(2)"><switch><foreignObject style="overflow: visible; text-align: left;" pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 118px; height: 1px; padding-top: 285px; margin-left: 211px;"><div style="box-sizing: border-box; font-size: 0px; text-align: center;" data-drawio-colors="color: rgb(0, 0, 0); "><div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: all; font-weight: bold; white-space: normal; overflow-wrap: normal;">Link Filter</div></div></div></foreignObject><text x="270" y="289" fill="rgb(0, 0, 0)" font-family="Helvetica" font-size="12px" text-anchor="middle" font-weight="bold">Link Filter</text></switch></g><rect x="420" y="630" width="240" height="60" rx="9" ry="9" fill="#dae8fc" stroke="#6c8ebf" stroke-width="2" pointer-events="all"/><g transform="translate(-0.5 -0.5)scale(2)"><switch><foreignObject style="overflow: visible; text-align: left;" pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 118px; height: 1px; padding-top: 330px; margin-left: 211px;"><div style="box-sizing: border-box; font-size: 0px; text-align: center;" data-drawio-colors="color: rgb(0, 0, 0); "><div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: all; font-weight: bold; white-space: normal; overflow-wrap: normal;">Parse Link</div></div></div></foreignObject><text x="270" y="334" fill="rgb(0, 0, 0)" font-family="Helvetica" font-size="12px" text-anchor="middle" font-weight="bold">Parse Link</text></switch></g><rect x="420" y="720" width="240" height="60" rx="9" ry="9" fill="#dae8fc" stroke="#6c8ebf" stroke-width="2" pointer-events="all"/><g transform="translate(-0.5 -0.5)scale(2)"><switch><foreignObject style="overflow: visible; text-align: left;" pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 118px; height: 1px; padding-top: 375px; margin-left: 211px;"><div style="box-sizing: border-box; font-size: 0px; text-align: center;" data-drawio-colors="color: rgb(0, 0, 0); "><div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: all; font-weight: bold; white-space: normal; overflow-wrap: normal;">Page Data</div></div></div></foreignObject><text x="270" y="379" fill="rgb(0, 0, 0)" font-family="Helvetica" font-size="12px" text-anchor="middle" font-weight="bold">Page Data</text></switch></g><path d="M 540 600 L 540 617.26" fill="none" stroke="rgb(0, 0, 0)" stroke-width="2" stroke-miterlimit="10" pointer-events="stroke"/><path d="M 540 627.76 L 533 613.76 L 540 617.26 L 547 613.76 Z" fill="rgb(0, 0, 0)" stroke="rgb(0, 0, 0)" stroke-width="2" stroke-miterlimit="10" pointer-events="all"/><path d="M 480 152 C 480 109.33 600 109.33 600 152 L 600 248 C 600 290.67 480 290.67 480 248 Z" fill="#dae8fc" stroke="#6c8ebf" stroke-width="2" stroke-miterlimit="10" pointer-events="all"/><path d="M 480 152 C 480 184 600 184 600 152" fill="none" stroke="#6c8ebf" stroke-width="2" stroke-miterlimit="10" pointer-events="all"/><g transform="translate(-0.5 -0.5)scale(2)"><switch><foreignObject style="overflow: visible; text-align: left;" pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 58px; height: 1px; padding-top: 112px; margin-left: 241px;"><div style="box-sizing: border-box; font-size: 0px; text-align: center;" data-drawio-colors="color: rgb(0, 0, 0); "><div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: all; font-weight: bold; white-space: normal; overflow-wrap: normal;">GenStage<br />Queue</div></div></div></foreignObject><text x="270" y="116" fill="rgb(0, 0, 0)" font-family="Helvetica" font-size="12px" text-anchor="middle" font-weight="bold">GenStage...</text></switch></g><path d="M 540 80 L 540 107.26" fill="none" stroke="rgb(0, 0, 0)" stroke-width="2" stroke-miterlimit="10" pointer-events="stroke"/><path d="M 540 117.76 L 533 103.76 L 540 107.26 L 547 103.76 Z" fill="rgb(0, 0, 0)" stroke="rgb(0, 0, 0)" stroke-width="2" stroke-miterlimit="10" pointer-events="all"/><path d="M 540 280 L 540 307.26" fill="none" stroke="rgb(0, 0, 0)" stroke-width="2" stroke-miterlimit="10" pointer-events="stroke"/><path d="M 540 317.76 L 533 303.76 L 540 307.26 L 547 303.76 Z" fill="rgb(0, 0, 0)" stroke="rgb(0, 0, 0)" stroke-width="2" stroke-miterlimit="10" pointer-events="all"/><path d="M 660 660 L 980 660 Q 1000 660 1000 640 L 1000 220 Q 1000 200 980 200 L 612.74 200" fill="none" stroke="rgb(0, 0, 0)" stroke-width="2" stroke-miterlimit="10" pointer-events="stroke"/><path d="M 602.24 200 L 616.24 193 L 612.74 200 L 616.24 207 Z" fill="rgb(0, 0, 0)" stroke="rgb(0, 0, 0)" stroke-width="2" stroke-miterlimit="10" pointer-events="all"/><path d="M 800 842 C 800 799.33 920 799.33 920 842 L 920 938 C 920 980.67 800 980.67 800 938 Z" fill="#dae8fc" stroke="#6c8ebf" stroke-width="2" stroke-miterlimit="10" pointer-events="all"/><path d="M 800 842 C 800 874 920 874 920 842" fill="none" stroke="#6c8ebf" stroke-width="2" stroke-miterlimit="10" pointer-events="all"/><g transform="translate(-0.5 -0.5)scale(2)"><switch><foreignObject style="overflow: visible; text-align: left;" pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 58px; height: 1px; padding-top: 457px; margin-left: 401px;"><div style="box-sizing: border-box; font-size: 0px; text-align: center;" data-drawio-colors="color: rgb(0, 0, 0); "><div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: all; font-weight: bold; white-space: normal; overflow-wrap: normal;">Data Store</div></div></div></foreignObject><text x="430" y="461" fill="rgb(0, 0, 0)" font-family="Helvetica" font-size="12px" text-anchor="middle" font-weight="bold">Data Store</text></switch></g><path d="M 740 776.52 L 840 776.09 Q 860 776 860 786.63 L 860 797.26" fill="none" stroke="rgb(0, 0, 0)" stroke-width="2" stroke-miterlimit="10" pointer-events="stroke"/><path d="M 860 807.76 L 853 793.76 L 860 797.26 L 867 793.76 Z" fill="rgb(0, 0, 0)" stroke="rgb(0, 0, 0)" stroke-width="2" stroke-miterlimit="10" pointer-events="all"/><path d="M 800 512 C 800 469.33 920 469.33 920 512 L 920 608 C 920 650.67 800 650.67 800 608 Z" fill="#dae8fc" stroke="#6c8ebf" stroke-width="2" stroke-miterlimit="10" pointer-events="all"/><path d="M 800 512 C 800 544 920 544 920 512" fill="none" stroke="#6c8ebf" stroke-width="2" stroke-miterlimit="10" pointer-events="all"/><g transform="translate(-0.5 -0.5)scale(2)"><switch><foreignObject style="overflow: visible; text-align: left;" pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 58px; height: 1px; padding-top: 292px; margin-left: 401px;"><div style="box-sizing: border-box; font-size: 0px; text-align: center;" data-drawio-colors="color: rgb(0, 0, 0); "><div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: all; font-weight: bold; white-space: normal; overflow-wrap: normal;">Disk</div></div></div></foreignObject><text x="430" y="296" fill="rgb(0, 0, 0)" font-family="Helvetica" font-size="12px" text-anchor="middle" font-weight="bold">Disk</text></switch></g><path d="M 700 380 L 747.27 379.21" fill="none" stroke="rgb(0, 0, 0)" stroke-width="2" stroke-miterlimit="10" stroke-dasharray="6 6" pointer-events="stroke"/><path d="M 757.76 379.04 L 743.88 386.27 L 747.27 379.21 L 743.65 372.27 Z" fill="rgb(0, 0, 0)" stroke="rgb(0, 0, 0)" stroke-width="2" stroke-miterlimit="10" pointer-events="all"/><rect x="440" y="380" width="200" height="40" rx="6" ry="6" fill="#e1d5e7" stroke="#9673a6" stroke-width="2" pointer-events="all"/><g transform="translate(-0.5 -0.5)scale(2)"><switch><foreignObject style="overflow: visible; text-align: left;" pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 98px; height: 1px; padding-top: 200px; margin-left: 221px;"><div style="box-sizing: border-box; font-size: 0px; text-align: center;" data-drawio-colors="color: rgb(0, 0, 0); "><div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: all; font-weight: bold; white-space: normal; overflow-wrap: normal;">Retry Strategy</div></div></div></foreignObject><text x="270" y="204" fill="rgb(0, 0, 0)" font-family="Helvetica" font-size="12px" text-anchor="middle" font-weight="bold">Retry Strategy</text></switch></g><rect x="20" y="20" width="220" height="220" fill="rgb(255, 255, 255)" stroke="rgb(0, 0, 0)" stroke-width="2" pointer-events="all"/><g transform="translate(-0.5 -0.5)scale(2)"><switch><foreignObject style="overflow: visible; text-align: left;" pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe flex-start; justify-content: unsafe center; width: 108px; height: 1px; padding-top: 17px; margin-left: 11px;"><div style="box-sizing: border-box; font-size: 0px; text-align: center;" data-drawio-colors="color: rgb(0, 0, 0); "><div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: all; white-space: normal; overflow-wrap: normal;"><font style="font-size: 10px">Legend</font></div></div></div></foreignObject><text x="65" y="29" fill="rgb(0, 0, 0)" font-family="Helvetica" font-size="12px" text-anchor="middle">Legend</text></switch></g><rect x="35.71" y="70.77" width="188.57" height="67.69" fill="#dae8fc" stroke="#6c8ebf" stroke-width="2" pointer-events="all"/><g transform="translate(-0.5 -0.5)scale(2)"><switch><foreignObject style="overflow: visible; text-align: left;" pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 92px; height: 1px; padding-top: 52px; margin-left: 19px;"><div style="box-sizing: border-box; font-size: 0px; text-align: center;" data-drawio-colors="color: rgb(0, 0, 0); "><div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: all; white-space: normal; overflow-wrap: normal;">Crawler Functionality</div></div></div></foreignObject><text x="65" y="56" fill="rgb(0, 0, 0)" font-family="Helvetica" font-size="12px" text-anchor="middle">Crawler Function...</text></switch></g><rect x="35.71" y="155.38" width="188.57" height="67.69" fill="#e1d5e7" stroke="#9673a6" stroke-width="2" pointer-events="all"/><g transform="translate(-0.5 -0.5)scale(2)"><switch><foreignObject style="overflow: visible; text-align: left;" pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 92px; height: 1px; padding-top: 95px; margin-left: 19px;"><div style="box-sizing: border-box; font-size: 0px; text-align: center;" data-drawio-colors="color: rgb(0, 0, 0); "><div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: all; white-space: normal; overflow-wrap: normal;">Customisable Functionality</div></div></div></foreignObject><text x="65" y="98" fill="rgb(0, 0, 0)" font-family="Helvetica" font-size="12px" text-anchor="middle">Customisable Fun...</text></switch></g><rect x="760" y="334" width="200" height="90" rx="13.5" ry="13.5" fill="#dae8fc" stroke="#6c8ebf" stroke-width="2" pointer-events="all"/><g transform="translate(-0.5 -0.5)scale(2)"><switch><foreignObject style="overflow: visible; text-align: left;" pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 98px; height: 1px; padding-top: 190px; margin-left: 381px;"><div style="box-sizing: border-box; font-size: 0px; text-align: center;" data-drawio-colors="color: rgb(0, 0, 0); "><div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: all; font-weight: bold; white-space: normal; overflow-wrap: normal;">Offline<br />File Handler</div></div></div></foreignObject><text x="430" y="193" fill="rgb(0, 0, 0)" font-family="Helvetica" font-size="12px" text-anchor="middle" font-weight="bold">Offline...</text></switch></g><path d="M 860 424 L 860 467.26" fill="none" stroke="rgb(0, 0, 0)" stroke-width="2" stroke-miterlimit="10" stroke-dasharray="6 6" pointer-events="stroke"/><path d="M 860 477.76 L 853 463.76 L 860 467.26 L 867 463.76 Z" fill="rgb(0, 0, 0)" stroke="rgb(0, 0, 0)" stroke-width="2" stroke-miterlimit="10" pointer-events="all"/><rect x="380" y="960" width="320" height="120" fill="#d5e8d4" stroke="#82b366" stroke-width="2" pointer-events="all"/><g transform="translate(-0.5 -0.5)scale(2)"><switch><foreignObject style="overflow: visible; text-align: left;" pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 158px; height: 1px; padding-top: 510px; margin-left: 191px;"><div style="box-sizing: border-box; font-size: 0px; text-align: center;" data-drawio-colors="color: rgb(0, 0, 0); "><div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: all; font-weight: bold; white-space: normal; overflow-wrap: normal;">Consumer Application<br /><font style="font-size: 10px">(with multiple ways of <br />consuming Crawler data)</font></div></div></div></foreignObject><text x="270" y="514" fill="rgb(0, 0, 0)" font-family="Helvetica" font-size="12px" text-anchor="middle" font-weight="bold">Consumer Application...</text></switch></g><path d="M 140 900 L 140 1000 Q 140 1020 160 1020 L 367.26 1020" fill="none" stroke="rgb(0, 0, 0)" stroke-width="2" stroke-miterlimit="10" stroke-dasharray="6 6" pointer-events="stroke"/><path d="M 377.76 1020 L 363.76 1027 L 367.26 1020 L 363.76 1013 Z" fill="rgb(0, 0, 0)" stroke="rgb(0, 0, 0)" stroke-width="2" stroke-miterlimit="10" pointer-events="all"/><path d="M 540 900 L 540 947.26" fill="none" stroke="rgb(0, 0, 0)" stroke-width="2" stroke-miterlimit="10" stroke-dasharray="6 6" pointer-events="stroke"/><path d="M 540 957.76 L 533 943.76 L 540 947.26 L 547 943.76 Z" fill="rgb(0, 0, 0)" stroke="rgb(0, 0, 0)" stroke-width="2" stroke-miterlimit="10" pointer-events="all"/><path d="M 860 970 L 860 1000 Q 860 1020 840 1020 L 712.74 1020" fill="none" stroke="rgb(0, 0, 0)" stroke-width="2" stroke-miterlimit="10" stroke-dasharray="6 6" pointer-events="stroke"/><path d="M 702.24 1020 L 716.24 1013 L 712.74 1020 L 716.24 1027 Z" fill="rgb(0, 0, 0)" stroke="rgb(0, 0, 0)" stroke-width="2" stroke-miterlimit="10" pointer-events="all"/><rect x="420" y="810" width="240" height="60" rx="9" ry="9" fill="#e1d5e7" stroke="#9673a6" stroke-width="2" pointer-events="all"/><g transform="translate(-0.5 -0.5)scale(2)"><switch><foreignObject style="overflow: visible; text-align: left;" pointer-events="none" width="100%" height="100%" requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"><div xmlns="http://www.w3.org/1999/xhtml" style="display: flex; align-items: unsafe center; justify-content: unsafe center; width: 118px; height: 1px; padding-top: 420px; margin-left: 211px;"><div style="box-sizing: border-box; font-size: 0px; text-align: center;" data-drawio-colors="color: rgb(0, 0, 0); "><div style="display: inline-block; font-size: 12px; font-family: Helvetica; color: rgb(0, 0, 0); line-height: 1.2; pointer-events: all; font-weight: bold; white-space: normal; overflow-wrap: normal;">Scraper</div></div></div></foreignObject><text x="270" y="424" fill="rgb(0, 0, 0)" font-family="Helvetica" font-size="12px" text-anchor="middle" font-weight="bold">Scraper</text></switch></g><path d="M 540 780 L 540 797.26" fill="none" stroke="rgb(0, 0, 0)" stroke-width="2" stroke-miterlimit="10" pointer-events="stroke"/><path d="M 540 807.76 L 533 793.76 L 540 797.26 L 547 793.76 Z" fill="rgb(0, 0, 0)" stroke="rgb(0, 0, 0)" stroke-width="2" stroke-miterlimit="10" pointer-events="all"/></g><switch><g requiredFeatures="http://www.w3.org/TR/SVG11/feature#Extensibility"/><a transform="translate(0,-5)" xlink:href="https://www.drawio.com/doc/faq/svg-export-text-problems" target="_blank"><text text-anchor="middle" font-size="10px" x="50%" y="100%">Text is not SVG - cannot display</text></a></switch></svg>

package/crawler/config/config.exs

@@ -0,0 +1,9 @@
+import Config
+
+config :logger,
+  backends: [:console],
+  compile_time_purge_matching: [[level_lower_than: :info]]
+
+if File.exists?("config/#{Mix.env()}.exs") do
+  import_config("#{Mix.env()}.exs")
+end

package/crawler/config/dev.exs

@@ -0,0 +1,5 @@
+import Config
+
+config :logger,
+  backends: [:console],
+  compile_time_purge_matching: [[level_lower_than: :debug]]

package/crawler/config/test.exs

@@ -0,0 +1,5 @@
+import Config
+
+config :logger,
+  backends: [:console],
+  compile_time_purge_matching: [[level_lower_than: :debug]]

package/crawler/examples/google_search/scraper.ex

@@ -0,0 +1,37 @@
+defmodule Crawler.Example.GoogleSearch.Scraper do
+  @moduledoc """
+  We only scrape Github pages, specifically looking for a project's name and description.
+  """
+
+  @behaviour Crawler.Scraper.Spec
+
+  alias Crawler.Store.Page
+  alias Crawler.Example.GoogleSearch.Data
+
+  def scrape(%Page{url: "https://github.com" <> _ = url, body: body, opts: _opts} = page) do
+    doc =
+      body
+      |> Floki.parse_document!()
+
+    name =
+      doc
+      |> Floki.find("#repository-container-header strong a")
+      |> Floki.text()
+
+    desc =
+      doc
+      |> Floki.find(".Layout-sidebar p.f4")
+      |> Floki.text()
+      |> String.trim()
+
+    if name != "" do
+      Agent.update(Data, fn state ->
+        Map.merge(state, %{name => %{url: url, desc: desc}})
+      end)
+    end
+
+    {:ok, page}
+  end
+
+  def scrape(page), do: {:ok, page}
+end

package/crawler/examples/google_search/url_filter.ex

@@ -0,0 +1,11 @@
+defmodule Crawler.Example.GoogleSearch.UrlFilter do
+  @moduledoc """
+  We start with Google, then only crawls Github.
+  """
+
+  @behaviour Crawler.Fetcher.UrlFilter.Spec
+
+  def filter("https://www.google.com" <> _, _opts), do: {:ok, true}
+  def filter("https://github.com" <> _, _opts), do: {:ok, true}
+  def filter(_url, _opts), do: {:ok, false}
+end

package/crawler/examples/google_search.ex

@@ -0,0 +1,77 @@
+defmodule Crawler.Example.GoogleSearch do
+  @moduledoc """
+  This example performs a Google search, then scrapes the results to find Github
+  projects and output their name and description.
+
+  Example output:
+
+      Agent.get(Data, & &1) #=> %{
+        "crawler" => %{
+          desc: "A high performance web crawler / scraper in Elixir.",
+          url: "https://github.com/fredwu/crawler"
+        },
+        "crawly" => %{
+          desc: "Crawly, a high-level web crawling & scraping framework for Elixir.",
+          url: "https://github.com/elixir-crawly/crawly"
+        },
+        "elixir_scraper" => %{
+          desc: "Elixir/Hound web scraper example",
+          url: "https://github.com/jaydorsey/elixir_scraper"
+        },
+        "mechanize" => %{
+          desc: "Build web scrapers and automate interaction with websites in Elixir with ease!",
+          url: "https://github.com/gushonorato/mechanize"
+        }
+      }
+  """
+
+  alias Crawler.Example.GoogleSearch.Data
+  alias Crawler.Example.GoogleSearch.Scraper
+  alias Crawler.Example.GoogleSearch.UrlFilter
+
+  @site_url "https://www.google.com/search?"
+  @search_term "github web scrapers in Elixir"
+
+  def run do
+    Agent.start_link(fn -> %{} end, name: Data)
+
+    # Do not crawl Google too fast, or you will get blocked
+
+    {:ok, opts} =
+      Crawler.crawl(
+        search_url(),
+        workers: 2,
+        max_depths: 2,
+        max_pages: 10,
+        interval: 80,
+        user_agent: "Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/118.0",
+        scraper: Scraper,
+        url_filter: UrlFilter
+      )
+
+    wait(fn ->
+      false = Crawler.running?(opts)
+
+      # give the scraper time to finish
+      Process.sleep(2_000)
+
+      dbg(Agent.get(Data, & &1))
+    end)
+  end
+
+  defp search_url do
+    @site_url <> URI.encode_query(%{"q" => @search_term})
+  end
+
+  defp wait(fun), do: wait(5_000, fun)
+
+  defp wait(timeout, fun) do
+    try do
+      fun.()
+    rescue
+      _ ->
+        :timer.sleep(500)
+        wait(max(0, timeout - 500), fun)
+    end
+  end
+end

package/crawler/lib/crawler/dispatcher/worker.ex

@@ -0,0 +1,14 @@
+defmodule Crawler.Dispatcher.Worker do
+  @moduledoc """
+  A worker that performs the crawling.
+  """
+
+  @doc """
+  Kicks off `Crawler.crawl_now/1`.
+  """
+  def start_link(opts) do
+    Task.start_link(fn ->
+      Crawler.crawl_now(opts)
+    end)
+  end
+end

package/crawler/lib/crawler/dispatcher.ex

@@ -0,0 +1,20 @@
+defmodule Crawler.Dispatcher do
+  @moduledoc """
+  Dispatches requests to a queue for crawling.
+  """
+
+  @doc """
+  Takes the `request` argument which is a tuple containing either:
+
+  - `{_, link, _, url}` when it's a link that got transformed into a URL
+  - `{_, url}` when it's a URL already
+
+  And issues `Crawler.crawl/2` to initiate the crawl.
+  """
+  def dispatch(request, opts) do
+    case request do
+      {_, _link, _, url} -> Crawler.crawl(url, opts)
+      {_, url} -> Crawler.crawl(url, opts)
+    end
+  end
+end

package/crawler/lib/crawler/fetcher/header_preparer.ex

@@ -0,0 +1,60 @@
+defmodule Crawler.Fetcher.HeaderPreparer do
+  @moduledoc """
+  Captures and prepares HTTP response headers.
+  """
+
+  @default_content_type "text/html"
+
+  @doc """
+  Captures and prepares HTTP response headers.
+
+  ## Examples
+
+      iex> HeaderPreparer.prepare(
+      iex>   [{"Content-Type", "text/html"}],
+      iex>   %{}
+      iex> )
+      %{headers: [{"Content-Type", "text/html"}], content_type: "text/html"}
+
+      iex> HeaderPreparer.prepare(
+      iex>   [{"Content-Type", "text/css"}],
+      iex>   %{}
+      iex> )
+      %{headers: [{"Content-Type", "text/css"}], content_type: "text/css"}
+
+      iex> HeaderPreparer.prepare(
+      iex>   [{"Content-Type", "image/png; blah"}],
+      iex>   %{}
+      iex> )
+      %{headers: [{"Content-Type", "image/png; blah"}], content_type: "image/png"}
+  """
+  def prepare(headers, opts) do
+    content_type =
+      headers
+      |> get_content_type()
+      |> simplify_content_type()
+
+    opts
+    |> Map.put(:headers, headers)
+    |> Map.put(:content_type, content_type)
+  end
+
+  defp get_content_type(nil), do: @default_content_type
+
+  defp get_content_type(headers) do
+    case Enum.find(headers, &find_content_type/1) do
+      {_, value} -> value
+      _ -> @default_content_type
+    end
+  end
+
+  defp find_content_type({header, _}) do
+    String.downcase(header) == "content-type"
+  end
+
+  defp simplify_content_type(content_type) do
+    content_type
+    |> String.split(";", parts: 2)
+    |> Kernel.hd()
+  end
+end

package/crawler/lib/crawler/fetcher/modifier.ex

@@ -0,0 +1,45 @@
+defmodule Crawler.Fetcher.Modifier do
+  @moduledoc """
+  Modifies request options and headers before dispatch.
+  """
+
+  defmodule Spec do
+    @type url :: String.t()
+    @type header :: {String.t(), String.t()}
+    @type opts :: map
+
+    @callback headers(opts) :: list(header) | []
+    @callback opts(opts) :: keyword | []
+  end
+
+  @behaviour __MODULE__.Spec
+
+  @doc """
+  Allows modifying headers prior to making a crawl request.
+
+  ## Example implementation
+
+      def headers(opts) do
+        if opts[:url] == "http://modifier" do
+          [{"Referer", "http://fetcher"}]
+        end
+        []
+      end
+  """
+  def headers(_opts), do: []
+
+  @doc """
+  Allows passing opts to httpPoison prior to making the crawl request
+
+  ## Example implementation
+
+      def opts(opts) do
+        if opts[:url] == "http://modifier" do
+          # add a new pool to hackney
+          [hackney: [pool: :modifier]]
+        end
+        []
+      end
+  """
+  def opts(_opts), do: []
+end

package/crawler/lib/crawler/fetcher/policer.ex

@@ -0,0 +1,77 @@
+defmodule Crawler.Fetcher.Policer do
+  @moduledoc """
+  Checks a series of conditions to determine whether it is okay to continue.
+  """
+
+  require Logger
+
+  alias Crawler.Store
+
+  @uri_schemes ["http", "https"]
+  @asset_extra_depth 2
+
+  @doc """
+  Checks a series of conditions to determine whether it is okay to continue,
+  i.e. to allow `Crawler.Fetcher.fetch/1` to begin its tasks.
+  """
+  def police(opts) do
+    with {_, true} <- within_max_pages?(opts),
+         {_, true} <- within_fetch_depth?(opts),
+         {_, true} <- acceptable_uri_scheme?(opts),
+         {_, true} <- not_fetched_yet?(opts),
+         {_, true} <- perform_url_filtering(opts) do
+      {:ok, opts}
+    else
+      {fail_type, _} -> police_warn(fail_type, opts)
+    end
+  end
+
+  defp within_max_pages?(%{max_pages: :infinity} = _opts), do: {:within_max_pages?, true}
+
+  defp within_max_pages?(%{max_pages: max_pages} = _opts) when is_integer(max_pages) do
+    {:within_max_pages?, Store.ops_count() < max_pages}
+  end
+
+  defp within_max_pages?(_opts), do: {:within_max_pages?, true}
+
+  defp within_fetch_depth?(%{depth: depth, max_depths: max_depths} = opts) do
+    max_depths =
+      case opts[:html_tag] do
+        "a" -> max_depths
+        _ -> max_depths + @asset_extra_depth
+      end
+
+    {:within_fetch_depth?, depth < max_depths}
+  end
+
+  defp within_fetch_depth?(_opts), do: {:within_fetch_depth?, true}
+
+  defp acceptable_uri_scheme?(%{url: url} = _opts) do
+    scheme =
+      url
+      |> String.split("://", parts: 2)
+      |> Kernel.hd()
+
+    {:acceptable_uri_scheme?, Enum.member?(@uri_schemes, scheme)}
+  end
+
+  defp acceptable_uri_scheme?(_opts), do: {:acceptable_uri_scheme?, true}
+
+  defp not_fetched_yet?(%{url: url, scope: scope} = _opts) do
+    {:not_fetched_yet?, !Store.find({url, scope})}
+  end
+
+  defp not_fetched_yet?(_opts), do: {:not_fetched_yet?, true}
+
+  defp perform_url_filtering(%{url_filter: url_filter, url: url} = opts) do
+    {:ok, pass_through?} = url_filter.filter(url, opts)
+
+    {:perform_url_filtering, pass_through?}
+  end
+
+  defp perform_url_filtering(_opts), do: {:perform_url_filtering, true}
+
+  defp police_warn(fail_type, opts) do
+    {:warn, "Fetch failed check '#{fail_type}', with opts: #{Kernel.inspect(opts)}."}
+  end
+end

package/crawler/lib/crawler/fetcher/recorder.ex

@@ -0,0 +1,55 @@
+defmodule Crawler.Fetcher.Recorder do
+  @moduledoc """
+  Records information about each crawl for internal use.
+  """
+
+  alias Crawler.Store
+
+  @doc """
+  Records information about each crawl for internal use.
+
+  ## Examples
+
+      iex> Recorder.record(url: "url1", depth: 2)
+      {:ok, %{depth: 3, url: "url1"}}
+
+      iex> Recorder.record(url: "url2", depth: 2)
+      iex> Store.find({"url2", nil})
+      %Page{url: "url2"}
+  """
+  def record(opts) do
+    with opts <- Enum.into(opts, %{}),
+         {:ok, _pid} <- store_url(opts),
+         opts <- store_url_depth(opts) do
+      {:ok, opts}
+    end
+  end
+
+  @doc """
+  Stores page data in `Crawler.Store.DB` for internal or external consumption, if enabled.
+
+  ## Examples
+
+      iex> Recorder.maybe_store_page("body", %{store: nil})
+      {:ok, nil}
+
+      iex> Recorder.record(url: "url", depth: 2)
+      iex> Recorder.maybe_store_page("body", %{store: Store, url: "url", scope: nil})
+      {:ok, {%Page{url: "url", body: "body", opts: %{store: Store, url: "url", scope: nil}}, %Page{url: "url", body: nil}}}
+  """
+  def maybe_store_page(_body, %{store: nil} = _opts) do
+    {:ok, nil}
+  end
+
+  def maybe_store_page(body, opts) do
+    {:ok, opts[:store].add_page_data({opts[:url], opts[:scope]}, body, opts)}
+  end
+
+  defp store_url(opts) do
+    Store.add({opts[:url], opts[:scope]})
+  end
+
+  defp store_url_depth(opts) do
+    Map.replace!(opts, :depth, opts[:depth] + 1)
+  end
+end

package/crawler/lib/crawler/fetcher/requester.ex

@@ -0,0 +1,32 @@
+defmodule Crawler.Fetcher.Requester do
+  @moduledoc """
+  Makes HTTP requests.
+  """
+
+  alias Crawler.HTTP
+
+  @fetch_opts [
+    follow_redirect: true,
+    max_redirect: 5
+  ]
+
+  @doc """
+  Makes HTTP requests via `Crawler.HTTP`.
+
+  ## Examples
+
+      iex> Requester.make(url: "fake.url", modifier: Crawler.Fetcher.Modifier)
+      {:error, %HTTPoison.Error{id: nil, reason: :nxdomain}}
+  """
+  def make(opts) do
+    HTTP.get(opts[:url], fetch_headers(opts), fetch_opts(opts))
+  end
+
+  defp fetch_headers(opts) do
+    [{"User-Agent", opts[:user_agent]}] ++ opts[:modifier].headers(opts)
+  end
+
+  defp fetch_opts(opts) do
+    @fetch_opts ++ [recv_timeout: opts[:timeout]] ++ opts[:modifier].opts(opts)
+  end
+end

package/crawler/lib/crawler/fetcher/retrier.ex

@@ -0,0 +1,43 @@
+defmodule Crawler.Fetcher.Retrier do
+  @moduledoc """
+  Handles retries for failed crawls.
+  """
+
+  defmodule Spec do
+    @moduledoc """
+    Spec for defining a fetch retrier.
+    """
+
+    @type fetch_url :: fun
+    @type opts :: map
+
+    @callback perform(fetch_url, opts) :: term
+  end
+
+  use Retry
+
+  @behaviour __MODULE__.Spec
+
+  @doc """
+  More information: [https://github.com/safwank/ElixirRetry](https://github.com/safwank/ElixirRetry)
+  """
+  def perform(fetch_url, opts) do
+    retry with:
+            exponential_backoff()
+            |> expiry(timeout_value(opts[:timeout]))
+            |> Stream.take(opts[:retries]) do
+      fetch_url.()
+    after
+      result -> result
+    else
+      error -> error
+    end
+  end
+
+  defp timeout_value(value) do
+    case Kernel.is_integer(value) do
+      true -> value
+      false -> 5_000
+    end
+  end
+end

package/crawler/lib/crawler/fetcher/url_filter.ex

@@ -0,0 +1,26 @@
+defmodule Crawler.Fetcher.UrlFilter do
+  @moduledoc """
+  A placeholder module that lets all URLs pass through.
+  """
+
+  defmodule Spec do
+    @moduledoc """
+    Spec for defining an url filter.
+    """
+
+    @type url :: String.t()
+    @type opts :: map
+
+    @callback filter(url, opts) :: {:ok, boolean} | {:error, term}
+  end
+
+  @behaviour __MODULE__.Spec
+
+  @doc """
+  Whether to pass through a given URL.
+
+  - `true` for letting the url through
+  - `false` for rejecting the url
+  """
+  def filter(_url, _opts), do: {:ok, true}
+end