webcheck-strings 1.601__tar.gz → 1.604__tar.gz

{webcheck_strings-1.601 → webcheck_strings-1.604}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: webcheck_strings
-Version: 1.601
+Version: 1.604
 Summary: Monitor text strings on websites
 Home-page: http://ssb22.user.srcf.net/setup/webcheck.html
 Author: Silas S. Brown

{webcheck_strings-1.601 → webcheck_strings-1.604}/setup.py

@@ -1,4 +1,4 @@
-from setuptools import setup, find_packages;setup(name='webcheck_strings',version='1.601',entry_points={'console_scripts':['webcheck=webcheck.__init__:main']},license='Apache 2',platforms='any',url='http://ssb22.user.srcf.net/setup/webcheck.html',author='Silas S. Brown',author_email='ssb22@cam.ac.uk',description='Monitor text strings on websites',long_description=r'''WebCheck
+from setuptools import setup, find_packages;setup(name='webcheck_strings',version='1.604',entry_points={'console_scripts':['webcheck=webcheck.__init__:main']},license='Apache 2',platforms='any',url='http://ssb22.user.srcf.net/setup/webcheck.html',author='Silas S. Brown',author_email='ssb22@cam.ac.uk',description='Monitor text strings on websites',long_description=r'''WebCheck
 ========
 
 From http://ssb22.user.srcf.net/setup/webcheck.html 

{webcheck_strings-1.601 → webcheck_strings-1.604}/webcheck/__init__.py

@@ -1,7 +1,8 @@
 #!/usr/bin/env python
 # (compatible with both Python 2 and Python 3)
 
-# webcheck.py v1.601 (c) 2014-24 Silas S. Brown.
+"""webcheck.py v1.604 (c) 2014-25 Silas S. Brown.
+License: Apache 2""" # (see below)
 # See webcheck.html for description and usage instructions
 
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -103,6 +104,7 @@ def read_input():
   days = 0 ; extraHeaders = []
   url = mainDomain = None
   lines = read_input_file()
+  if not lines: print ("No input found")
   lastList = None # refs a list within ret, of [(days,text,elseLogic)]
   while lines:
     line = line_withComment = " ".join(lines.pop().split())
@@ -271,7 +273,7 @@ class Delayer:
   def __init__(self,mainDomain):
     self.last_fetch_finished = 0
     if mainDomain=="stackoverflow.com":
-      # (or other Stack Exchange sites)
+      # (or other Stack Exchange sites as normalised to stackoverflow.com above) (although in 2024-12 I was able to confirm that Stack Exchange's own email notifications of others' edits to your answers _did_ work, so it might not be necessary to monitor for vandalism all your Stack Exchange answers from webcheck)
       self.delay = 80 # seconds between fetches to these sites
     else: self.delay = 3 # (want small if checking many pages and the server doesn't mind, but still non-0)
   def wait(self):
@@ -403,9 +405,9 @@ def doJob(opener,delayer,url,checklist,extraHeaders):
       textContent = None
   delayer.done()
   if content==None: return # not modified (so nothing to report), or problem retrieving (which will have been reported by tryRead0: TODO: return failRet in these circumstances so elseLogic can proceed?)
-  if B(content).startswith(B("<!DOCTYPE html>")):
+  if B(content).startswith(B("<!DOCTYPE html>")) or B(content).startswith(B('<html lang=')): # check for CloudFlare backoff
     textContent,errmsg=htmlStrings(content)
-    if B(textContent)==B("Just a moment... Enable JavaScript and cookies to continue"): raise CDNBackoff() # might not be able to check the rest of this domain today (at least not without starting a webdriver, but for things like 'checking if my StackExchange answers have been vandalised' the site probably doesn't want too many automated fetches; would be easier if they provided an RSS feed for each user)
+    if B(textContent).startswith(B("Just a moment...")) and B("Enable JavaScript and cookies to continue") in B(textContent): raise CDNBackoff() # might not be able to check the rest of this domain today
   if u:
       lm = u.info().get("Last-Modified",None)
       if lm: previous_timestamps[(url,'lastMod')] = lm
@@ -466,6 +468,8 @@ def run_webdriver(ua,actionList,reportErrors):
         return B(""), True
     r = "" ; wasError = False
     try: r = run_webdriver_inner(actionList,browser)
+    except CDNBackoff:
+      browser.quit() ; raise
     except NoTracebackException as e:
       if reportErrors: print (e.message)
       else: wasError = True
@@ -503,16 +507,20 @@ def run_webdriver_inner(actionList,browser):
       return f(browser).encode('utf-8')
     snippets = []
     for a in actionList:
-        if a.startswith('http'): browser.get(a)
+        if a.startswith('http'):
+          try: browser.get(a)
+          except: raise NoTracebackException("webdriver low-level timeout or error fetching "+a) # e.g. selenium.common.exceptions.TimeoutException
         elif a.startswith('"') and a.endswith('"'):
             # wait for "string" to appear in the source
             tries = 30
             while tries and not myFind(a[1:-1],getSrc()):
               time.sleep(2) ; tries -= 1
             if not tries:
+              src = getSrc() ; h=B(htmlStrings(src)[0])
+              if h.startswith(B("Just a moment...")) and B("Enable JavaScript and cookies to continue") in h: raise CDNBackoff() # 2024-07: can occur even in webdriver, even though that has JS + cookies
               try: current_url = browser.current_url
               except: current_url = "(unable to obtain)"
-              raise NoTracebackException("webdriver timeout while waiting for %s, current URL is %s content \"%s\"\n" % (repr(a[1:-1]),current_url,repr(getSrc()))) # don't quote current URL: if the resulting email is viewed in (at least some versions of) MHonArc, a bug can result in &quot being added to the href
+              raise NoTracebackException("webdriver timeout while waiting for %s, current URL is %s content \"%s\"\n" % (repr(a[1:-1]),current_url,repr(src))) # don't quote current URL: if the resulting email is viewed in (at least some versions of) MHonArc, a bug can result in &quot being added to the href
         elif a.startswith('[') and a.endswith(']'): # click
             findElem(a[1:-1]).click()
         elif a.startswith('/') and '/' in a[1:]: # click through items in a list to reveal each one (assume w/out Back)
@@ -545,7 +553,7 @@ def run_webdriver_inner(actionList,browser):
                   if sys.stderr.isatty(): sys.stderr.write('*'),sys.stderr.flush()
                 except:
                   if sys.stderr.isatty(): sys.stderr.write('?'),sys.stderr.flush()
-                  if propagate_errors: raise NoTracebackException(a+" failed to open instance "+str(curNo))
+                  if propagate_errors: raise NoTracebackException(a+" failed to open instance "+str(curNo)+" (error because ! is set)")
                   else: continue
                 time.sleep(delayAfter)
                 snippets.append(getSrc())
@@ -569,7 +577,7 @@ def run_webdriver_inner(actionList,browser):
                 if sys.stderr.isatty(): sys.stderr.write('*'),sys.stderr.flush() # webdriver's '.' for click-multiple
               except:
                 if sys.stderr.isatty(): sys.stderr.write('?'),sys.stderr.flush()
-                if propagate_errors: raise NoTracebackException(a+" failed to open instance "+str(curNo))
+                if propagate_errors: raise NoTracebackException(a+" failed to open instance "+str(curNo)+" (error because ! is set)")
                 else: continue
               time.sleep(delayAfter)
               snippets.append(getSrc())
@@ -843,4 +851,7 @@ def normalisePunc(t):
       ]: t=t.replace(s,r)
   return re.sub(B(r"(\s)\s+"),B(r"\1"),t).lower()
 
-if __name__=="__main__": main()
+if __name__=="__main__":
+  if "--version" in sys.argv: print(__doc__)
+  elif "--help" in sys.argv: print("python webcheck.py [--single-thread] [--test-all] [--show-seen-rss] | --help | --version")
+  else: main()

{webcheck_strings-1.601 → webcheck_strings-1.604}/webcheck_strings.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: webcheck-strings
-Version: 1.601
+Version: 1.604
 Summary: Monitor text strings on websites
 Home-page: http://ssb22.user.srcf.net/setup/webcheck.html
 Author: Silas S. Brown