pywb 2.7.2__tar.gz → 2.7.4__tar.gz

{pywb-2.7.2 → pywb-2.7.4}/CHANGES.rst

@@ -1,3 +1,19 @@
+pywb 2.7.3 changelist
+~~~~~~~~~~~~~~~~~~~~~
+
+* issue_792 catch warcio exception by @oskarhek in https://github.com/webrecorder/pywb/pull/793
+* Add ui.logo_home_url as config.yaml option by @tw4l in https://github.com/webrecorder/pywb/pull/791
+* [#795] Show error when adding duplicate warc file by @kuechensofa in https://github.com/webrecorder/pywb/pull/797
+* Make search page more intuitive by @krakan in https://github.com/webrecorder/pywb/pull/794
+* Modify search template buttons by @tw4l in https://github.com/webrecorder/pywb/pull/801
+* [#804] Use default_locale when lang not set in the request by @krakan in https://github.com/webrecorder/pywb/pull/805
+* feat: regex substitution on surt rules match by @mijho in https://github.com/webrecorder/pywb/pull/780
+* Bump minimatch from 3.0.4 to 3.1.2 in /pywb/vueui by @dependabot in https://github.com/webrecorder/pywb/pull/777
+* Bump decode-uri-component from 0.2.0 to 0.2.2 in /pywb/vueui by @dependabot in https://github.com/webrecorder/pywb/pull/786
+* rules: add 'debugNoBatch' rewrite for fb and insta by @ikreymer in https://github.com/webrecorder/pywb/pull/806
+* Vue main order by @tw4l in https://github.com/webrecorder/pywb/pull/809
+* wombat: bump to 3.4.4 https://github.com/webrecorder/pywb/pull/808
+
 pywb 2.7.2 changelist
 ~~~~~~~~~~~~~~~~~~~~~
 

{pywb-2.7.2/pywb.egg-info → pywb-2.7.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: pywb
-Version: 2.7.2
+Version: 2.7.4
 Summary: Pywb Webrecorder web archive replay and capture tools
 Home-page: https://github.com/webrecorder/pywb
 Author: Ilya Kreymer

{pywb-2.7.2 → pywb-2.7.4}/build-vue-ui.sh

@@ -3,4 +3,5 @@
 CURR_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
 
 cd $CURR_DIR/pywb/vueui/
+yarn install
 yarn run build

{pywb-2.7.2 → pywb-2.7.4}/config.yaml

@@ -6,6 +6,7 @@ debug: true
 # Uncomment to set banner colors and logo
 # ui:
   # logo: path/relative/from/static/logo.png
+  # logo_home_url: https://example.com
   # navbar_background_hex: 0c49b0
   # navbar_color_hex: fff
   # navbar_light_buttons: true

{pywb-2.7.2 → pywb-2.7.4}/docs/manual/vue-ui.rst

@@ -53,6 +53,21 @@ For example, to use the file ``./static/my-logo.png`` as the logo, set:
     logo: my-logo.png
 
 
+Logo URL
+^^^^^^^^
+
+It is possible to configure the logo to link to any URL by setting ``ui.logo_home_url`` in ``config.yml`` to the URL of your choice.
+
+If omitted, the logo will not link to any page.
+
+For example, to have the logo redirect to ``https://example.com/web-archive-landing-page``, set:
+
+.. code:: yaml
+
+  ui:
+    logo_home_url: https://example.com/web-archive-landing-page
+
+
 Banner Colors
 ^^^^^^^^^^^^^
 

{pywb-2.7.2 → pywb-2.7.4}/pywb/apps/frontendapp.py

@@ -108,6 +108,7 @@ class FrontEndApp(object):
         self.templates_dir = config.get('templates_dir', 'templates')
         self.static_dir = config.get('static_dir', 'static')
         self.static_prefix = config.get('static_prefix', 'static')
+        self.default_locale = config.get('default_locale', '')
 
         metadata_templ = os.path.join(self.warcserver.root_dir, '{coll}', 'metadata.yaml')
         self.metadata_cache = MetadataCache(metadata_templ)
@@ -662,7 +663,7 @@ class FrontEndApp(object):
             # store original script_name (original prefix) before modifications are made
             environ['ORIG_SCRIPT_NAME'] = environ.get('SCRIPT_NAME')
 
-            lang = args.pop('lang', '')
+            lang = args.pop('lang', self.default_locale)
             if lang:
                 pop_path_info(environ)
                 environ['pywb_lang'] = lang

{pywb-2.7.2 → pywb-2.7.4}/pywb/apps/rewriterapp.py

@@ -248,8 +248,8 @@ class RewriterApp(object):
         range_start = start
         range_end = end
 
-        # if start with 0, load from upstream, but add range after
-        if start == 0:
+        # if start with 0 and no end, load from upstream
+        if start == 0 and not end:
             del inputreq.env['HTTP_RANGE']
         else:
             skip_record = True

pywb-2.7.4/pywb/git_hash.py

@@ -0,0 +1 @@
+git_hash = "b7ea9af8"

{pywb-2.7.2 → pywb-2.7.4}/pywb/indexer/cdxindexer.py

@@ -1,5 +1,9 @@
+import logging
 import os
 import sys
+import traceback
+
+import warcio
 
 # Use ujson if available
 try:
@@ -298,8 +302,11 @@ def write_multi_cdx_index(output, inputs, **options):
                 with open(fullpath, 'rb') as infile:
                     entry_iter = record_iter(infile)
 
-                    for entry in entry_iter:
-                        writer.write(entry, filename)
+                    try:
+                        for entry in entry_iter:
+                            writer.write(entry, filename)
+                    except warcio.exceptions.ArchiveLoadFailed:
+                        logging.error('Error while indexing file %s, %s',filename,traceback.format_exc())
 
         return writer
 

{pywb-2.7.2 → pywb-2.7.4}/pywb/manager/manager.py

@@ -121,14 +121,24 @@ directory structure expected by pywb
                           format(self.archive_dir))
 
         full_paths = []
+        duplicate_warcs = []
         for filename in warcs:
             filename = os.path.abspath(filename)
+
+            # don't overwrite existing warcs with duplicate names
+            if os.path.exists(os.path.join(self.archive_dir, os.path.basename(filename))):
+                duplicate_warcs.append(filename)
+                continue
+
             shutil.copy2(filename, self.archive_dir)
             full_paths.append(os.path.join(self.archive_dir, filename))
             logging.info('Copied ' + filename + ' to ' + self.archive_dir)
 
         self._index_merge_warcs(full_paths, self.DEF_INDEX_FILE)
 
+        if duplicate_warcs:
+            logging.warning(f'Warcs {", ".join(duplicate_warcs)} weren\'t added because of duplicate names.')
+
     def reindex(self):
         cdx_file = os.path.join(self.indexes_dir, self.DEF_INDEX_FILE)
         logging.info('Indexing ' + self.archive_dir + ' to ' + cdx_file)

{pywb-2.7.2 → pywb-2.7.4}/pywb/rules.yaml

@@ -110,7 +110,7 @@ rules:
 
       fuzzy_lookup:
         match: '("(?:cursor|cursorindex)":["\d\w]+)'
-        find_all: true
+        re_type: findall
 
     - url_prefix: 'com,facebook)/ajax/pagelet/generic.php/profiletimeline'
       fuzzy_lookup: 'com,facebook\)/.*[?&](__adt=[^&]+).*[&]data=(?:.*?(?:[&]|(profile_id|pagelet_token)[^,]+))'
@@ -175,7 +175,7 @@ rules:
 
       fuzzy_lookup:
         match: '("q[\d]+":|after:\\"[^"]+)'
-        find_all: true
+        re_type: findall
 
     - url_prefix: 'com,facebook)/pages_reaction_units/more'
 
@@ -196,6 +196,9 @@ rules:
               group: 1
               function: 'pywb.rewrite.rewrite_dash:rewrite_fb_dash'
 
+            - match: '"debugNoBatching\s?":(?:false|0)'
+              replace: '"debugNoBatching":true'
+
         parse_comments: true
 
     - url_prefix: 'com,facebook'
@@ -227,6 +230,9 @@ rules:
             - match: '"is_dash_eligible":true'
               replace: '"is_dash_eligible":false'
 
+            - match: '"debugNoBatching\s?":(?:false|0)'
+              replace: '"debugNoBatching":true'
+
       fuzzy_lookup: '()'
 
 
@@ -538,6 +544,12 @@ rules:
       rewrite:
         js_rewrite_location: urls
 
+    - url_prefix: 'com,example)/matched'
+      fuzzy_lookup:
+        re_type: sub
+        match: 'matched'
+        replace: 'replaced'          
+
     # all domain rules -- fallback to this dataset
     #=================================================================
     # Applies to all urls -- should be last

{pywb-2.7.2 → pywb-2.7.4}/pywb/static/query.js

@@ -956,11 +956,11 @@ RenderCalendar.prototype.niceDateRange = function() {
   var from = this.queryInfo.searchParams.from;
   var to = this.queryInfo.searchParams.to;
   if (from && to) {
-    return 'From ' + from + ' to ' + to;
+    return [text.from, from, text.until, to].join(' ');
   } else if (from) {
-    return 'From ' + from + ' until ' + 'present';
+    return [text.from, from, text.until, text.present].join(' ');
   }
-  return 'From earliest until ' + to;
+  return [text.from, text.earliest, text.until, to].join(' ');
 };
 
 /**

{pywb-2.7.2 → pywb-2.7.4}/pywb/static/search.js

@@ -14,17 +14,34 @@ var elemIds = {
   },
   dateTime: {
     from: 'dt-from',
+    fromTime: 'ts-from',
     fromBad: 'dt-from-bad',
     to: 'dt-to',
+    toTime: 'ts-to',
     toBad: 'dt-to-bad'
   },
   match: 'match-type-select',
   url: 'search-url',
   form: 'search-form',
   resultsNewWindow: 'open-results-new-window',
-  advancedOptions: 'advanced-options'
+  advancedOptions: 'advanced-options',
+  resetSearchForm: 'reset-search-form',
 };
 
+function resetSearchForm(event) {
+  for (const field of [
+    elemIds.url,
+    elemIds.match,
+    elemIds.dateTime.from,
+    elemIds.dateTime.fromTime,
+    elemIds.dateTime.to,
+    elemIds.dateTime.toTime,
+  ]) {
+    document.getElementById(field).value = '';
+  }
+  clearFilters(event);
+}
+
 function makeCheckDateRangeChecker(dtInputId, dtBadNotice) {
   var dtInput = document.getElementById(dtInputId);
   dtInput.onblur = function() {
@@ -138,11 +155,13 @@ function performQuery(url) {
   }
   var fromT = document.getElementById(elemIds.dateTime.from).value;
   if (fromT) {
-    query.push('from=' + fromT.trim());
+    fromT += document.getElementById(elemIds.dateTime.fromTime).value;
+    query.push('from=' + fromT.replace(/[^0-9]/g, ''));
   }
   var toT = document.getElementById(elemIds.dateTime.to).value;
   if (toT) {
-    query.push('to=' + toT.trim());
+    toT += document.getElementById(elemIds.dateTime.toTime).value;
+    query.push('to=' + toT.replace(/[^0-9]/g, ''));
   }
   var builtQuery = query.join('&');
   if (document.getElementById(elemIds.resultsNewWindow).checked) {
@@ -188,6 +207,7 @@ $(document).ready(function() {
     elemIds.dateTime.to,
     document.getElementById(elemIds.dateTime.toBad)
   );
+  document.getElementById(elemIds.resetSearchForm).onclick = resetSearchForm;
   document.getElementById(elemIds.filtering.add).onclick = addFilter;
   document.getElementById(elemIds.filtering.clear).onclick = clearFilters;
   var searchURLInput = document.getElementById(elemIds.url);
@@ -195,9 +215,6 @@ $(document).ready(function() {
   form.addEventListener('submit', function(event) {
     submitForm(event, form, searchURLInput);
   });
-  document.getElementById(elemIds.advancedOptions).onclick = function() {
-    validateFields(form);
-  }
   var filteringExpression = document.getElementById(elemIds.filtering.expression);
   filteringExpression.addEventListener("keypress", function(event) {
     if (event.key === "Enter") {