py2ls 0.1.4.2__py3-none-any.whl → 0.1.4.5__py3-none-any.whl

{py2ls-0.1.4.2.dist-info → py2ls-0.1.4.5.dist-info}/RECORD

@@ -1,5 +1,6 @@
-py2ls/.git/COMMIT_EDITMSG,sha256=pQUXyLHaZO274VHn0hpa4QarY5RJacPcvzWdQH-QYcM,71
-py2ls/.git/FETCH_HEAD,sha256=dEddBzNY8P7TDi5FxO4cpwtS-_PitMWUxom-CfUr5dw,100
+py2ls/.DS_Store,sha256=1lFlJ5EFymdzGAUAaI30vcaaLHt3F1LwpG7xILf9jsM,6148
+py2ls/.git/COMMIT_EDITMSG,sha256=411ztgfI0SRPtVH7bl1HcadUmgTDrnUC7vUpClHTuhg,22
+py2ls/.git/FETCH_HEAD,sha256=JftrmaQ9tRhDaUKGGYHLcduOOWrCUcIXNgXZnFtj7MI,100
 py2ls/.git/HEAD,sha256=KNJb-Cr0wOK3L1CVmyvrhZ4-YLljCl6MYD2tTdsrboA,21
 py2ls/.git/config,sha256=XswTg1Ts7_7IBDlKHh4OF_0Tq7v4wW7BXb6xSVInSec,345
 py2ls/.git/description,sha256=ZzMxc0Ca26m45Twn1DDnOHqin5VHEZ9uOTBrScIXSjE,16
@@ -16,34 +17,47 @@ py2ls/.git/hooks/pre-receive.sample,sha256=pMPSuce7P9jRRBwxvU7nGlldZrRPz0ndsxAlI
 py2ls/.git/hooks/prepare-commit-msg.sample,sha256=6d3KpBif3dJe2X_Ix4nsp7bKFjkLI5KuMnbwyOGqRhk,1492
 py2ls/.git/hooks/push-to-checkout.sample,sha256=pT0HQXmLKHxt16-mSu5HPzBeZdP0lGO7nXQI7DsSv18,2783
 py2ls/.git/hooks/update.sample,sha256=jV8vqD4QPPCLV-qmdSHfkZT0XL28s32lKtWGCXoU0QY,3650
-py2ls/.git/index,sha256=hMDaY5LEl9oBKXChb9Y1W0VHPaAYzXErgaSun_5Dfvw,1338
+py2ls/.git/index,sha256=ELDmOGBjKw_yg4T29FlGfjGCtOPyp-0HeobCvVn2FEk,1242
 py2ls/.git/info/exclude,sha256=ZnH-g7egfIky7okWTR8nk7IxgFjri5jcXAbuClo7DsE,240
-py2ls/.git/logs/HEAD,sha256=v2lK-ld92AZRnRMM8iB5eQRDBT6SISXYmwjMfGvbepk,1297
-py2ls/.git/logs/refs/heads/main,sha256=v2lK-ld92AZRnRMM8iB5eQRDBT6SISXYmwjMfGvbepk,1297
-py2ls/.git/logs/refs/remotes/origin/HEAD,sha256=0J6k2shFsGg9lj6WgUFDQCxo8HMVyCod44BhjrUUk5E,1782
-py2ls/.git/logs/refs/remotes/origin/main,sha256=1opXMlnx9w8Tj0G9_RkMjK-CXZv553s67VfcFS3G36U,1216
+py2ls/.git/logs/HEAD,sha256=atxmhKxlt0NtNNva_ke0lzelzyiplpA7eq8VWbYCppU,2283
+py2ls/.git/logs/refs/heads/main,sha256=atxmhKxlt0NtNNva_ke0lzelzyiplpA7eq8VWbYCppU,2283
+py2ls/.git/logs/refs/remotes/origin/HEAD,sha256=PdHzZi_OInxBkrCeCoKkc_wYjhAi8CYJcHdb1ZBskbw,5931
+py2ls/.git/logs/refs/remotes/origin/main,sha256=BwjVsMFWPHygHFr2LkjMCNoxelgIAOBSd0SgxIEnOy4,1976
 py2ls/.git/objects/0b/409e1bc918277010f5679b402d1d1dda53e15c,sha256=y5S1XaGxJz1NXi-SPWjPC_NKIqqSbZv9oOg74MzBihY,156
 py2ls/.git/objects/14/449a0e6ba4ea2f1a73acf63ef91c9c6193f9ed,sha256=PomZFmCUCQM1ii0wH-OJGSHLQCTqRtIwE5w3C0TtzSY,171
 py2ls/.git/objects/15/a8e468aacfcb440e090020f36d0b985d45da23,sha256=xiRunMcN5I_B2hHgBUFupR-F0b8H_CQTmmAZG9XkZik,3215
+py2ls/.git/objects/1a/b4585881a6a42889f01aa0cfe25fd5acfaf46f,sha256=iQsKMPNKUs4WQwhiLgXmG5V3xKyIgxmc13ZwbBATvhQ,165
 py2ls/.git/objects/1d/fe9d9633b24ea560354f4f93d39c6e5f163ea0,sha256=mV_84wLqIitnSYmzfrNpTzwVP9AmksiRI0Fjltwl0Pg,8872
 py2ls/.git/objects/24/6b368b986f758630c46dc02b7fa512b53422f7,sha256=sw7ERFCFu7m6fnURAqQfQ4GWShaARr-Vc6GRnlOPkxU,8512
 py2ls/.git/objects/25/b796accd261b9135fd32a2c00785f68edf6c46,sha256=4ic5vOwEdfbGL8oARSVEeAnSoDs14-gggGZEL-61nYE,564
 py2ls/.git/objects/30/a2f8da47ee947811dc8d993f5a06a45de374f4,sha256=u5W33_qNtTs1-U8Fardx-zB_udqKvuCm5kiw1mQGdsU,3218
 py2ls/.git/objects/36/b4a1b7403abc6c360f8fe2cb656ab945254971,sha256=X18sHo17gQTxusNptoILoWxSfU0ulrsZvbETlBtV5aY,2327
+py2ls/.git/objects/36/e56a361f526eafa59c5235a5c990bf288b5f9c,sha256=7L1L-iqVvuufrlfEE2myD0-QnAPueFMySKetu08W-Pc,34216
 py2ls/.git/objects/36/ef43e50009e59db11812c258846d9e38718173,sha256=0nwCwQSuQAdGyD2NfEK-_L12ydE9nGVKBXOfFq_Lndc,169
 py2ls/.git/objects/3b/bd972aa7ad680858f8dfbd0f7fcd97756f0d6f,sha256=MQWEzML3wbb4GixiHDCHrxgbXLQitrYDstT1plhmQSU,169
 py2ls/.git/objects/3c/bbe5f4173d165127b9ad96119f1ec24c306ffc,sha256=S1BXemROYtzRaj5WXLPYnTmPTBQDKovMEN0GRLul-I4,33489
 py2ls/.git/objects/3f/d6561300938afbb3d11976cf9c8f29549280d9,sha256=91oqbTWfUE1d_hT_1ptYmRUb5pOQ1X4oxQxpF6NXjKU,8501
+py2ls/.git/objects/43/dbd49b2ee367c5434dd545e3b5795434f2ef0b,sha256=DAzt0dWp2KsuuImCKp7N9ia7KaCDNqwB-tYIx3Wf_c0,565
 py2ls/.git/objects/48/a88fc5806305d0bb0755ee6801161b79696972,sha256=f3JStE39k_hPGE-WRwqZtDTjQkfOmBVb_6-ELBbScjI,203
+py2ls/.git/objects/50/08ddfcf53c02e82d7eee2e57c38e5672ef89f6,sha256=p0M2WLqiTe6X2FI_k5Aj0IEsE85jqLa58sVdmV8x1vU,255
+py2ls/.git/objects/53/e0deb1cb4c2c606bced6e7f9a66b0fda60980d,sha256=muq6m7_XRSFPzypW-m9mhpKfsomCr4s7GfkgM3gh2pc,482344
+py2ls/.git/objects/56/e4e8b2d5545e0256090f45aa8fc42c5fe067d0,sha256=VsjKo1biAzCV-iIfwCDTPzyfP63K43hdZqJpDP70Iik,529
 py2ls/.git/objects/58/20a729045d4dc7e37ccaf8aa8eec126850afe2,sha256=3Pf6NS8OTK4EdHZGVeJ421BtK7w4WJncQDBauZI_wW4,34
 py2ls/.git/objects/60/f273eb1c412d916fa3f11318a7da7a9911b52a,sha256=aJD9iF_LmYSrqDepXFBZKN1yMYbQczVkN_wnrDosBdI,5620
 py2ls/.git/objects/61/570cec8c061abe74121f27f5face6c69b98f99,sha256=IQZi5MkbRu3ToRUPsRcXuh1Xa3pkAz_HDRCVhNL89ds,5753
+py2ls/.git/objects/62/7c81b23b4e56e87b042b650b0103653cc9e34a,sha256=pv9wgBxnvJUFSrk9G7vApA6lnSykQSMJ4yXT7YnlSDU,167
+py2ls/.git/objects/62/d90ccf8cbefdc2e4fd475e7c6f4f76e9fdf801,sha256=1L473QanNpnumCkE8tG6wtbvLqFtNeoagL9SJmasXNY,155
+py2ls/.git/objects/64/27a4edff08f93d98f511418423f09f2ab90bcd,sha256=RyNngwk9fvdvvvywmNfllnim718fWNjVauH9U2y8Q2s,258
 py2ls/.git/objects/69/13c452ca319f7cbf6a0836dc10a5bb033c84e4,sha256=NYLQQZTfd0htZst42ALS2dmryv1q_l1N19ZfHEbz_38,3193
 py2ls/.git/objects/6b/7fde264d93a7a0986d394c46c7650d0ce2ab92,sha256=iIl0-RF0wd6BSEjzczgUyApxc899PbdTl04JbDn6_-Q,166
+py2ls/.git/objects/6c/cebb29b7f3f5b0c889f6dadbf9ff066554587d,sha256=UylkFWAfhStNVQRQuC9CzpaWaT9uHCVs1mn7ecOma8I,609
 py2ls/.git/objects/78/3d4167bc95c9d2175e0df03ef1c1c880ba75ab,sha256=SK2QDjDBiDhVMG1I5p19g4RbEm2Rax7mYnxawmVZYxs,15523
 py2ls/.git/objects/79/7ae089b2212a937840e215276005ce76881307,sha256=lQOKF2pb1JvipI3eT79X0-TuMGWsy1A-Yw4BCgKZNOM,33472
 py2ls/.git/objects/7e/5956c806b5edc344d46dab599dec337891ba1f,sha256=sfqJBiSNj-gyJo4D7xkmRAo76mC2ztjqeZZsl4ifULA,162
+py2ls/.git/objects/81/8f26b7bf042269729020cf944fc362d66ba27e,sha256=mg6FGEyv6EcOgurR8CEvHGovaWrUgMUxTtACAy7-ei4,34960
+py2ls/.git/objects/84/59071b722a255b774a80b27746033f8141ab39,sha256=0pYGJOXFfp4MSu4n5MzE1XN--t0lSs7wcdqboADWMx0,9792
 py2ls/.git/objects/87/ef1fc3f7f1ddc4d0ab9b3e65381ce9f3388621,sha256=OFrpW6lu31qGBvD3ijPUBSG9JrdU1_mKzeYBzidn9VM,3748
+py2ls/.git/objects/8b/84f56978e1de8f2ae82abce5f8b3e182d365cd,sha256=a8XequnUMBSv9zIQJdcdgDvMQ7PLGdIrgZ-MqQGF87c,573
 py2ls/.git/objects/8e/55a7d2b96184030211f20c9b9af201eefcac82,sha256=yW-jVYeCTWR-nX3JJgA1g9YLPjzNsKlDmEOH290Ywx0,1221
 py2ls/.git/objects/91/c69ad88fe0ba94aa7859fb5f7edac5e6f1a3f7,sha256=Kk2MWCO1OcShYuABGzp2O9LiWGDfDkcZtd0oy4nY6RU,9529
 py2ls/.git/objects/9d/0df52899fe95279059286d9c0ec42287edc168,sha256=67nV3TLo-fwe4lt0wwvxoDnVNHc1IpapRyAY2STP3iI,564
@@ -52,35 +66,42 @@ py2ls/.git/objects/a7/3e13eafee65c5b8d73ad2d3ea46d0eee82f0d3,sha256=iv3uTzna5XBz
 py2ls/.git/objects/b0/56be4be89ba6b76949dd641df45bb7036050c8,sha256=8Y7z30eNceBd5QIx09QfMp5cYBbrgUllmats0kvJEJ4,132
 py2ls/.git/objects/b0/9cd7856d58590578ee1a4f3ad45d1310a97f87,sha256=82dx4hIdMpdcB64e5PU1s2gZFVkTvrj1cPwwJ_kasNU,4444
 py2ls/.git/objects/b2/18e6a0f0f1c4df8cdefa9852058348abc713b7,sha256=hOQfdyzDZctjoge0-pAcEDel5XHVPNfOtrMNyFPUOIE,564
+py2ls/.git/objects/b5/61831c7dce8ea51e7ee6b6fa35745f14d8242d,sha256=wUqxlKjLN1vOUj2tkYStado64QewdcF3CHlSICds1ik,34415
+py2ls/.git/objects/bb/934eb33bc1a8b85630bf680caffd99560c1b8f,sha256=ggehjexUsWlskHJvHxW7u6U0otB0OCItmIZdT9O-3OU,9670
 py2ls/.git/objects/c4/cba65f1163661999ee4b8ed23342b63bc1300c,sha256=rwSdKt-C98nUQ_B-7imY4fYRYmn29MQc4SIu9wruHeo,566
+py2ls/.git/objects/c6/7f17e5707313600efcb85e9a3fedea35dba591,sha256=TL7rDIWiaWlk8iIwqPst7St5Xr2otPs-vp17GPlET7o,565
+py2ls/.git/objects/cf/0c0d9c6fb09473aaeb7f7e2edbd770c3f2ef3d,sha256=T_nV0GrgpVu3mOJ4fYcCW98oCunzgqy0DnSX0luy04Q,183
+py2ls/.git/objects/d6/9ab1c4aadf279936dd778e8346ba60f74705b6,sha256=WcfdSMKqfiWT5TOWVUcDj0XDaD2hYxDnyIRNlYGutL8,34976
 py2ls/.git/objects/d9/005f2cc7fc4e65f14ed5518276007c08cf2fd0,sha256=IJIoz93V7pf9yx43U1JdN8gBq_LWtw8A9Z2YMPnq_B0,1450
 py2ls/.git/objects/d9/c2403fd166ce791b4e9d0c6792ed8342c71fcd,sha256=uD7BsKdrmN-9FStTpwsRWh-XxVXeDsV4dGjFkaMIIs8,170
+py2ls/.git/objects/d9/dfa5aee51e92a541b707e8e7baea6f06deff98,sha256=jMdhZ1i_L5q_UgjOtjLN15PCSCz3pE51FhD3z74ZUr8,163
 py2ls/.git/objects/db/ffa8ea7bda721d0cee7b9e4ce5b2ef927733ff,sha256=GhDkvP6JYV26qVg5ETPys1ZEnGlsct9hiXCc24Ky4Xg,565
 py2ls/.git/objects/df/e0770424b2a19faf507a501ebfc23be8f54e7b,sha256=vCdlxwEidekh8i-5TVMVgSLGk9DPZCZAbWqvGYSKQ9c,76
+py2ls/.git/objects/e3/1356f90ea6dd0577b5e0b40b206319adcbf085,sha256=I9_QNwmmtoqSwq29Ixdfv_PgF2x14u2M6sX1eQumwoY,161
 py2ls/.git/objects/e3/5a4dafc50850cacac7bf76c56db2715cbda2c4,sha256=GAcBj3YSEbm6tm7fGD6al16uBo8LtEtjZ2Hi-UgIsUg,3290
 py2ls/.git/objects/e9/391ffe371f1cc43b42ef09b705d9c767c2e14f,sha256=RWTy2n8L2XxZQknBFyPczA0Aa_4gSG_Ybcr8e8v4ccc,10264
 py2ls/.git/objects/f4/b64d3107b39e3ad6f540c6607004ea34e6c024,sha256=0egAtqc0x8hc7U1z91tIjcRhSd_BT2a_gxZxo_7NTJA,564
 py2ls/.git/objects/f7/c98ba5c2f903e603b1f5e63d49fbc8a43815cc,sha256=tYbi3A7irrIPB_11bwItuof0Vc9a0MDuLFMNAzRsG3A,33467
+py2ls/.git/objects/fa/147e6bb78a2e8db241d231295fd7f1ed061af8,sha256=G9pg5LXv7AdxnPIQsTm2AF3Un314dLRJQYwxmZem9rQ,574
 py2ls/.git/objects/fc/292e793ecfd42240ac43be407023bd731fa9e7,sha256=hGIYoxKWNT3IPwk3DE4l3FLBbUYF-kXcHcx7KrH9uS0,1971
-py2ls/.git/refs/heads/main,sha256=b_JnHFFXWnMz1M-3Lc6pKdZvxu7ZGWRaSTn-1i008Ro,41
+py2ls/.git/refs/heads/main,sha256=3RjrJjIDvFj75BVJA5LA1QcuJulTJWk6nGYinmb21q4,41
 py2ls/.git/refs/remotes/origin/HEAD,sha256=K7aiSqD8bEhBAPXVGim7rYQc0sdV9dk_qiBOXbtOsrQ,30
-py2ls/.git/refs/remotes/origin/main,sha256=b_JnHFFXWnMz1M-3Lc6pKdZvxu7ZGWRaSTn-1i008Ro,41
+py2ls/.git/refs/remotes/origin/main,sha256=lZgord5ZoIag3eQnw5oqNUSn85Nh7qyeconduoPcMls,41
 py2ls/.gitattributes,sha256=Gh2-F2vCM7SZ01pX23UT8pQcmauXWfF3gwyRSb6ZAFs,66
 py2ls/.gitignore,sha256=y7GvbD_zZkjPVVIue8AyiuFkDMuUbvMaV65Lgu89To8,2763
 py2ls/LICENSE,sha256=UOZ1F5fFDe3XXvG4oNnkL1-Ecun7zpHzRxjp-XsMeAo,11324
 py2ls/README.md,sha256=CwvJWAnSXnCnrVHlnEbrxxi6MbjbE_MT6DH2D53S818,11572
-py2ls/__init__.py,sha256=k05sUEKwWnAaOAn4VKCiPopcWrbmLLsZOuAt7vmUKWc,260
+py2ls/__init__.py,sha256=ESXjQ9tnqg5mqYH4Gfgs76AoT1HHF_BkJUgnstiVwR8,243
 py2ls/brain_atlas.py,sha256=w1o5EelRjq89zuFJUNSz4Da8HnTCwAwDAZ4NU4a-bAY,5486
 py2ls/correlators.py,sha256=RbOaJIPLCHJtUm5SFi_4dCJ7VFUPWR0PErfK3K26ad4,18243
 py2ls/dbhandler.py,sha256=i9dNrpHyx0oIaFieHI4X4tsrCdN-aFxudPTDOgy9Ppo,3574
 py2ls/freqanalysis.py,sha256=F4218VSPbgL5tnngh6xNCYuNnfR-F_QjECUUxrPYZss,32594
-py2ls/internet_finder.py,sha256=I-jPepbBhpDaOAsD2MqbKMe1CBN8w1PYo0CjNFkaeeU,19149
-py2ls/ips.py,sha256=mrn1V-7p7Xpv-9p0clP2HzY3Jo1NN5pd-nhbycE-y0U,119243
-py2ls/netfinder.py,sha256=IliFR4vsiG-07a0JSCUYtXlxIA9AXiK8QMdbBToFv1M,30693
+py2ls/ips.py,sha256=yf4UECX0bhzyoKqyRWBOGUrNrzu-BoO2f8G293eR2Do,124079
+py2ls/netfinder.py,sha256=dt6hkYeH-ivCHInoUi92MhJMLlXtjRXT3ewKzOwGtWk,31506
+py2ls/setuptools-70.1.0-py3-none-any.whl,sha256=2bi3cUVal8ip86s0SOvgspteEF8SKLukECi-EWmFomc,882588
 py2ls/sleep_events_detectors.py,sha256=36MCuRrpurn0Uvzpo3p3b3_JlVsRNHSWCXbJxCGM3mg,51546
 py2ls/translator.py,sha256=QfDUO0-pXHGMBFZBefiBHzOrC93-__N5sUQY_VP4wes,29734
-py2ls/version.py,sha256=CactNZqrHHYTPrkHKccy2WKXmaiUdtTgPqSjFyVXnJk,18
 py2ls/wb_detector.py,sha256=7y6TmBUj9exCZeIgBAJ_9hwuhkDh1x_-yg4dvNY1_GQ,6284
-py2ls-0.1.4.2.dist-info/METADATA,sha256=E1ANItbpXa2-LfGfDj0X_AzUxinK_U_0L_zXfd2eREs,11877
-py2ls-0.1.4.2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-py2ls-0.1.4.2.dist-info/RECORD,,
+py2ls-0.1.4.5.dist-info/METADATA,sha256=Lbz_REQylOCgSwziBFP6oEELfCEU2cK01PCM4kKo5MI,17943
+py2ls-0.1.4.5.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+py2ls-0.1.4.5.dist-info/RECORD,,

py2ls/internet_finder.py

@@ -1,405 +0,0 @@
-from bs4 import BeautifulSoup
-import requests
-import os
-from urllib.parse import urlparse, urljoin
-import base64
-import pandas as pd
-from collections import Counter
-import random
-import logging
-from time import sleep
-import stem.process
-from stem import Signal
-from stem.control import Controller
-import json
-# Set up logging
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger(__name__)
-
-# Define supported content types and corresponding parsers
-CONTENT_PARSERS = {
-    "text/html": lambda text, parser: BeautifulSoup(text, parser),
-    "application/json": lambda text, parser: json.loads(text),
-    "text/xml": lambda text, parser: BeautifulSoup(text, parser),
-    "text/plain": lambda text, parser: text.text,
-}
-
-def fetch_all(url, parser="lxml"): # lxml is faster, # parser="html.parser"
-    try:
-        # Generate a random user-agent string
-        headers = {"User-Agent": user_agent()}
-
-        # Send the initial request
-        response = requests.get(url, headers=headers)
-
-        # If the response is a redirect, follow it
-        while response.is_redirect:
-            logger.info(f"Redirecting to: {response.headers['Location']}")
-            response = requests.get(response.headers["Location"], headers=headers)
-        # Check for a 403 error
-        if response.status_code == 403:
-            logger.warning("403 Forbidden error. Retrying...")
-            # Retry the request after a short delay
-            sleep(random.uniform(1, 3))
-            response = requests.get(url, headers=headers)
-            # Raise an error if retry also fails
-            response.raise_for_status()
-
-        # Raise an error for other HTTP status codes
-        response.raise_for_status()
-
-        # Get the content type
-        content_type = response.headers.get("content-type", "").split(";")[0].lower()
-        content = response.content.decode(response.encoding)
-        # logger.info(f"Content type: {content_type}")
-
-        # Check if content type is supported
-        if content_type in CONTENT_PARSERS:
-            return content_type, CONTENT_PARSERS[content_type](content, parser)
-        else:
-            logger.warning("Unsupported content type")
-            return None, None
-    except requests.RequestException as e:
-        logger.error(f"Error fetching URL '{url}': {e}")
-        return None, None
-def user_agent():
-    # Example of generating a random user-agent string
-    user_agents = [
-        # Windows (Intel)
-        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4891.0 Safari/537.36",
-        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4893.0 Safari/537.36",
-        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4895.0 Safari/537.36",
-        # Windows (ARM)
-        "Mozilla/5.0 (Windows NT 10.0; Win64; arm64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4891.0 Safari/537.36",
-        "Mozilla/5.0 (Windows NT 10.0; Win64; arm64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4893.0 Safari/537.36",
-        "Mozilla/5.0 (Windows NT 10.0; Win64; arm64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4895.0 Safari/537.36",
-        # Linux (x86_64)
-        "Mozilla/5.0 (X11; Linux x86_64; rv:98.0) Gecko/20100101 Firefox/98.0",
-        "Mozilla/5.0 (X11; Linux x86_64; rv:99.0) Gecko/20100101 Firefox/99.0",
-        "Mozilla/5.0 (X11; Linux x86_64; rv:100.0) Gecko/20100101 Firefox/100.0",
-        "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.36",
-        "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4891.0 Safari/537.36",
-        "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4893.0 Safari/537.36",
-        "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4895.0 Safari/537.36",
-        "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:98.0) Gecko/20100101 Firefox/98.0",
-        "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:99.0) Gecko/20100101 Firefox/99.0",
-        "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:100.0) Gecko/20100101 Firefox/100.0",
-        # macOS (Intel)
-        "Mozilla/5.0 (Macintosh; Intel Mac OS X 12_0_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.3 Safari/605.1.15",
-        "Mozilla/5.0 (Macintosh; Intel Mac OS X 12_0_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.4 Safari/605.1.15",
-        # macOS (ARM)
-        "Mozilla/5.0 (Macintosh; ARM Mac OS X 12_0_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.3 Safari/605.1.15",
-        "Mozilla/5.0 (Macintosh; ARM Mac OS X 12_0_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.4 Safari/605.1.15",
-        # iOS Devices
-        "Mozilla/5.0 (iPad; CPU OS 15_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.0 Mobile/15E148 Safari/604.1",
-        "Mozilla/5.0 (iPhone; CPU iPhone OS 15_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.0 Mobile/15E148 Safari/604.1",
-        # Android Devices
-        "Mozilla/5.0 (Linux; Android 12; Pixel 6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4891.0 Mobile Safari/537.36",
-        "Mozilla/5.0 (Linux; Android 12; Pixel 6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4893.0 Mobile Safari/537.36",
-        "Mozilla/5.0 (Linux; Android 12; Pixel 6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4895.0 Mobile Safari/537.36",
-        # Smart TVs
-        "Mozilla/5.0 (SMART-TV; LINUX; Tizen 6.0) AppleWebKit/537.36 (KHTML, like Gecko) SmartTV/1.0",
-        "Mozilla/5.0 (SMART-TV; LINUX; Tizen 6.0) AppleWebKit/537.36 (KHTML, like Gecko) WebAppManager/1.0",
-        # Game Consoles
-        "Mozilla/5.0 (PlayStation 5 3.01) AppleWebKit/605.1.15 (KHTML, like Gecko)",
-        "Mozilla/5.0 (Xbox One 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.182 Safari/537.36 Edge/44.18363.8740",
-    ]
-    agents = random.choice(user_agents)
-    return agents
-
-# # Function to change Tor IP address
-# def renew_tor_ip():
-#     with Controller.from_port(port=9051) as controller:
-#         controller.authenticate()
-#         controller.signal(Signal.NEWNYM)
-
-# # Function to make requests through Tor
-# def make_tor_request(url, max_retries=3):
-#     renew_tor_ip()
-#     headers = {"User-Agent": user_agent()}
-#     session = requests.Session()
-#     session.proxies = {"http": "socks5h://localhost:9050", "https": "socks5h://localhost:9050"}
-
-#     for i in range(max_retries):
-#         try:
-#             response = session.get(url, headers=headers, timeout=10)
-#             if response.status_code == 200:
-#                 return response.text
-#         except requests.exceptions.RequestException as e:
-#             print(f"Error: {e}")
-#         time.sleep(2)  # Add a delay between retries
-
-#     return None
-
-
-def find_links(url):
-    links_href = []  # Initialize list to store extracted links
-    content_type, content = fetch_all(url)
-    base_url = urlparse(url)
-    links = content.find_all("a", href=True)
-    for link in links:
-        link_href = link["href"]
-        if not link_href.startswith(('http://', 'https://')):
-            # Convert relative links to absolute links
-            link_href = urljoin(base_url.geturl(), link_href)
-        links_href.append(link_href)
-    return links_href
-
-def find_domain(links):
-    domains = [urlparse(link).netloc for link in links]
-    domain_counts = Counter(domains)
-    most_common_domain = domain_counts.most_common(1)[0][0]
-    # print(f"Most_frequent_domain:{most_common_domain}")
-    return most_common_domain
-
-# To determine which links are related to target domains(e.g., pages) you are interested in
-def filter_links(links, domain=None, kind='html'):
-    filtered_links = []
-    if isinstance(kind, (str, list)):
-        kind = tuple(kind)
-    if domain is None:
-        domain = find_domain(links)
-    for link in links:
-        parsed_link = urlparse(link)
-        if parsed_link.netloc == domain and parsed_link.path.endswith(kind) and 'javascript:' not in parsed_link:
-            filtered_links.append(link)
-    return filtered_links
-
-def find_img(url, dir_save="images"):
-    """
-    Save images referenced in HTML content locally.
-    Args:
-        content (str or BeautifulSoup): HTML content or BeautifulSoup object.
-        url (str): URL of the webpage.
-        content_type (str): Type of content. Default is "html".
-        dir_save (str): Directory to save images. Default is "images".
-    Returns:
-        str: HTML content with updated image URLs pointing to local files.
-    """
-    content_type, content = fetch_all(url)
-    if "html" in content_type.lower():
-        # Create the directory if it doesn't exist
-        os.makedirs(dir_save, exist_ok=True)
-
-        # Parse HTML content if it's not already a BeautifulSoup object
-        if isinstance(content, str):
-            content = BeautifulSoup(content, "html.parser")
-        image_links=[]
-        # Extracting images
-        images = content.find_all("img", src=True)
-        for i, image in enumerate(images):
-            try:
-                # Get the image URL
-                image_url = image["src"]
-
-                if image_url.startswith("data:image"):
-                    # Extract the image data from the data URI
-                    mime_type, base64_data = image_url.split(",", 1)
-                    # Determine the file extension from the MIME type
-                    if ":" in mime_type:
-                        # image_extension = mime_type.split(":")[1].split(";")[0]
-                        image_extension = mime_type.split(":")[1].split(";")[0].split("/")[-1]
-                    else:
-                        image_extension = "png"  # Default to PNG if extension is not specified
-                    # if 'svg+xml' in image_extension:
-                    #     image_extension='svg'
-                    image_data = base64.b64decode(base64_data)
-                    # Save the image data to a file
-                    image_filename = os.path.join(
-                        dir_save, f"image_{i}.{image_extension}"
-                    )
-                    with open(image_filename, "wb") as image_file:
-                        image_file.write(image_data)
-
-                    # Update the src attribute of the image tag to point to the local file
-                    image["src"] = image_filename
-                else:
-                    # Construct the absolute image URL
-                    absolute_image_url = urljoin(url, image_url)
-
-                    # Parse the image URL to extract the file extension
-                    parsed_url = urlparse(absolute_image_url)
-                    image_extension = os.path.splitext(parsed_url.path)[1]
-
-                    # Download the image
-                    image_response = requests.get(absolute_image_url)
-
-                    # Save the image to a file
-                    image_filename = os.path.join(
-                        dir_save, f"image_{i}{image_extension}"
-                    )
-                    with open(image_filename, "wb") as image_file:
-                        image_file.write(image_response.content)
-
-                    # Update the src attribute of the image tag to point to the local file
-                    image["src"] = image_filename
-            except (requests.RequestException, KeyError) as e:
-                print(f"Failed to process image {image_url}: {e}")
-        print(f"images were saved at\n{dir_save}")
-    # Return the HTML content with updated image URLs
-    return content
-
-def content_div_class(content, div="div", div_class="highlight"):
-    texts = [div.text for div in content.find_all(div, class_=div_class)]
-    return texts
-def find(url, where="div", what="highlight"):
-    _,content = fetch_all(url, parser="html.parser")
-    texts = [div.text for div in content.find_all(where, class_=what)]
-    return texts
-# usage example:
-#### img2local(url, "/Users/macjianfeng/Desktop/@tmp/dd/")
-def find_forms(url):
-    content_type, content = fetch_all(url)
-    df=pd.DataFrame()
-    # Extracting forms and inputs
-    forms = content.find_all("form")
-    form_data = []
-    for form in forms:
-        form_inputs = form.find_all("input")
-        input_data = {}
-        for input_tag in form_inputs:
-            input_type = input_tag.get("type")
-            input_name = input_tag.get("name")
-            input_value = input_tag.get("value")
-            input_data[input_name] = {"type": input_type, "value": input_value}
-        form_data.append(input_data)
-    return form_data
-#  to clean strings
-def clean_string(value):
-    if isinstance(value, str):
-        return value.replace('\n', '').replace('\r', '').replace('\t', '')
-    else:
-        return value
-def find_all(url, dir_save=None):
-    content_type, content = fetch_all(url)
-
-    # Extracting paragraphs
-    paragraphs_text = [paragraph.text for paragraph in content.find_all("p")]
-
-    # Extracting specific elements by class
-    specific_elements_text = [element.text for element in content.find_all(class_="specific-class")]
-
-    # Extracting links (anchor tags)
-    links_href = find_links(url)
-    links_href = filter_links(links_href)
-
-    # Extracting images
-    images_src = [image['src'] for image in content.find_all("img", src=True)]
-
-    # Extracting headings (h1, h2, h3, etc.)
-    headings = [f'h{i}' for i in range(1, 7)]
-    headings_text = {heading: [tag.text for tag in content.find_all(heading)] for heading in headings}
-
-    # Extracting lists (ul, ol, li)
-    list_items_text = [item.text for list_ in content.find_all(["ul", "ol"]) for item in list_.find_all("li")]
-
-    # Extracting tables (table, tr, td)
-    table_cells_text = [cell.text for table in content.find_all("table") for row in table.find_all("tr") for cell in row.find_all("td")]
-
-    # Extracting other elements
-    divs_content = [div.text.strip() for div in content.find_all("div")]
-    headers_footer_content = [tag.text for tag in content.find_all(["header", "footer"])]
-    meta_tags_content = [(tag.name, tag.attrs) for tag in content.find_all("meta")]
-    spans_content = [span.text for span in content.find_all("span")]
-    bold_text_content = [text.text for text in content.find_all("b")]
-    italic_text_content = [text.text for text in content.find_all("i")]
-    code_snippets_content = [code.text for code in content.find_all("code")]
-    blockquotes_content = [blockquote.text for blockquote in content.find_all("blockquote")]
-    preformatted_text_content = [pre.text for pre in content.find_all("pre")]
-    buttons_content = [button.text for button in content.find_all("button")]
-    navs_content = [nav.text for nav in content.find_all("nav")]
-    sections_content = [section.text for section in content.find_all("section")]
-    articles_content = [article.text for article in content.find_all("article")]
-    figures_content = [figure.text for figure in content.find_all("figure")]
-    captions_content = [caption.text for caption in content.find_all("figcaption")]
-    abbreviations_content = [abbr.text for abbr in content.find_all("abbr")]
-    definitions_content = [definition.text for definition in content.find_all("dfn")]
-    addresses_content = [address.text for address in content.find_all("address")]
-    time_elements_content = [time.text for time in content.find_all("time")]
-    progress_content = [progress.text for progress in content.find_all("progress")]
-    meter_content = [meter.text for meter in content.find_all("meter")]
-    forms = find_forms(url)
-
-    lists_to_fill = [
-        paragraphs_text, specific_elements_text, links_href, images_src,
-        headings_text["h1"], headings_text["h2"], headings_text["h3"], headings_text["h4"],
-        headings_text["h5"], headings_text["h6"], list_items_text, table_cells_text,
-        divs_content, headers_footer_content, meta_tags_content, spans_content,
-        bold_text_content, italic_text_content, code_snippets_content,
-        blockquotes_content, preformatted_text_content, buttons_content,
-        navs_content, sections_content, articles_content, figures_content,
-        captions_content, abbreviations_content, definitions_content,
-        addresses_content, time_elements_content, progress_content,
-        meter_content,forms
-    ]
-    # add new features
-    script_texts=content_div_class(content, div="div", div_class="highlight")
-    lists_to_fill.append(script_texts)
-    
-    audio_src = [audio['src'] for audio in content.find_all("audio", src=True)]
-    video_src = [video['src'] for video in content.find_all("video", src=True)]
-    iframe_src = [iframe['src'] for iframe in content.find_all("iframe", src=True)]
-    lists_to_fill.extend([audio_src, video_src, iframe_src])
-    
-    rss_links = [link['href'] for link in content.find_all('link', type=['application/rss+xml', 'application/atom+xml'])]
-    lists_to_fill.append(rss_links)
-
-    # Find the maximum length among all lists
-    max_length = max(len(lst) for lst in lists_to_fill)
-
-    # Fill missing data with empty strings for each list
-    for lst in lists_to_fill:
-        lst += [""] * (max_length - len(lst))
-
-    # Create DataFrame
-    df = pd.DataFrame({
-        "headings1": headings_text["h1"],
-        "headings2": headings_text["h2"],
-        "headings3": headings_text["h3"],
-        "headings4": headings_text["h4"],
-        "headings5": headings_text["h5"],
-        "headings6": headings_text["h6"],
-        "paragraphs": paragraphs_text,
-        "list_items": list_items_text,
-        "table_cells": table_cells_text,
-        "headers_footer": headers_footer_content,
-        "meta_tags": meta_tags_content,
-        "spans": spans_content,
-        "bold_text": bold_text_content,
-        "italic_text": italic_text_content,
-        "code_snippets": code_snippets_content,
-        "blockquotes": blockquotes_content,
-        "preformatted_text": preformatted_text_content,
-        "buttons": buttons_content,
-        "navs": navs_content,
-        "sections": sections_content,
-        "articles": articles_content,
-        "figures": figures_content,
-        "captions": captions_content,
-        "abbreviations": abbreviations_content,
-        "definitions": definitions_content,
-        "addresses": addresses_content,
-        "time_elements": time_elements_content,
-        "progress": progress_content,
-        "specific_elements": specific_elements_text,
-        "meter": meter_content,
-        "forms":forms,
-        "scripts":script_texts,
-        "audio":audio_src,
-        "video":video_src,
-        "iframe":iframe_src,
-        "rss": rss_links,
-        "images": images_src,
-        "links": links_href,
-        "divs": divs_content,
-    })
-    # to remove the '\n\t\r'
-    df=df.apply(lambda x: x.map(clean_string) if x.dtype == "object" else x) # df=df.applymap(clean_string)
-    if dir_save:
-        if not dir_save.endswith(".csv"):
-            dir_save=dir_save+"_df.csv"
-            df.to_csv(dir_save)
-        else:
-            df.to_csv(dir_save)
-        print(f"file has been saved at\n{dir_save}")
-    return df

py2ls/version.py

@@ -1 +0,0 @@
-version = "0.0.1"