docassemble.InterviewStats 0.5.0__tar.gz → 0.6.0__tar.gz

docassemble_interviewstats-0.6.0/MANIFEST.in

@@ -0,0 +1,18 @@
+include README.md
+graft docassemble/InterviewStats/data
+recursive-exclude * *.egg-info
+recursive-exclude .git *
+recursive-exclude venv *
+recursive-exclude .github *
+recursive-exclude .pytest_cache *
+recursive-exclude .vscode *
+recursive-exclude build *
+recursive-exclude dist *
+recursive-exclude * __pycache__
+recursive-exclude * *.pyc
+recursive-exclude * *.pyo
+recursive-exclude * *.orig
+recursive-exclude * *~
+recursive-exclude * *.bak
+recursive-exclude * *.swp
+

{docassemble.InterviewStats-0.5.0/docassemble.InterviewStats.egg-info → docassemble_interviewstats-0.6.0}/PKG-INFO

@@ -1,16 +1,21 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: docassemble.InterviewStats
-Version: 0.5.0
-Summary: A docassemble extension, to view stats from other interviews
-Home-page: https://courtformsonline.org/about/
-Author: Bryce Willey, Quinten Steenhuis
-Author-email: bwilley@suffolk.com
-License: The MIT License (MIT)
+Version: 0.6.0
+Summary: A docassemble interview to view statistics from other saved interview responses.
+Author-email: Suffolk Legal Innovation and Technology Lab <litlab@suffolk.edu>
+License-Expression: MIT
+Project-URL: GitHub, https://github.com/SuffolkLITLab/docassemble-InterviewStats
 Description-Content-Type: text/markdown
 License-File: LICENSE
+Requires-Dist: numpy>=1.0.4
+Requires-Dist: pandas>=1.4.2
+Requires-Dist: requests>=2.27.1
+Dynamic: license-file
 
 # docassemble.InterviewStats
 
+[![PyPI version](https://badge.fury.io/py/docassemble.InterviewStats.svg)](https://badge.fury.io/py/docassemble.InterviewStats)
+
 A docassemble interview that lets you view statistics from other saved interview responses. 
 
 ## Getting Started
@@ -51,20 +56,10 @@ visiting this url:
 They will be directed immediately to download an XlSX file containing the
 statistics.
 
-## Example
-
-![Example Pic](static/example_pic.png)
-
-
-## Roadmap
-
-Currently, we can show simple grouping over all the data points in the interview stats. However, the
-special visualization are only shown on based on the `zip` attribute. We're working to expand the
-features available here in conjunction with our [EFiling Integration](https://github.com/SuffolkLITLab/EfileProxyServer).
+## Contributing 
 
 If you have specific feature requests, feel free to open an issue or make a PR!
 
-
 ## Authors
 
 * Quinten Steenhuis

{docassemble.InterviewStats-0.5.0 → docassemble_interviewstats-0.6.0}/README.md

@@ -1,5 +1,7 @@
 # docassemble.InterviewStats
 
+[![PyPI version](https://badge.fury.io/py/docassemble.InterviewStats.svg)](https://badge.fury.io/py/docassemble.InterviewStats)
+
 A docassemble interview that lets you view statistics from other saved interview responses. 
 
 ## Getting Started
@@ -40,20 +42,10 @@ visiting this url:
 They will be directed immediately to download an XlSX file containing the
 statistics.
 
-## Example
-
-![Example Pic](static/example_pic.png)
-
-
-## Roadmap
-
-Currently, we can show simple grouping over all the data points in the interview stats. However, the
-special visualization are only shown on based on the `zip` attribute. We're working to expand the
-features available here in conjunction with our [EFiling Integration](https://github.com/SuffolkLITLab/EfileProxyServer).
+## Contributing 
 
 If you have specific feature requests, feel free to open an issue or make a PR!
 
-
 ## Authors
 
 * Quinten Steenhuis

docassemble_interviewstats-0.6.0/docassemble/InterviewStats/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.6.0"

{docassemble.InterviewStats-0.5.0 → docassemble_interviewstats-0.6.0}/docassemble/InterviewStats/data/questions/data_report.yml

@@ -37,6 +37,7 @@ code: |
 #	Question screens
 #--------------------------------------------------------------------------
 ---
+id: data-reporting
 question: |
   Data Reporting
 subquestion: |
@@ -55,6 +56,7 @@ fields:
     code: |      
       [{interview.get('filename'): interview.get('title')} for interview in interview_menu()] or get_filenames()
 ---
+id: quick-summary-screen
 question: |
   Quick Summary for ${ interview_title }
 subquestion: |
@@ -65,10 +67,8 @@ subquestion: |
   % if summary_stats[0] > 10000:
   You have a lot of sessions; some operations might take a long time.
   % endif  
-  ---
-  Do you want to filter your data records by date?
 fields:      
-  - no label: filter_date
+  - Do you want to filter your data records by date?: filter_date
     datatype: yesnoradio
     default: False
     required: False
@@ -117,15 +117,17 @@ subquestion: |
   The following fields are from single-valued questions, except those whose answer lack statistic value, such as user-typed answer or numeric input. 
   
   Select a field to see its table:<br>
-    % for col in col_headings_single:  
-      [${col}](${ url_action('show_filtered_data_single', filter_by = col)  })
+    % for col_single in col_headings_single:  
+      [${col_single}](${ url_action('show_filtered_data_single', filter_by = col_single)  })
     % endfor
   % endif
   
   #### All questions (excluding text/area/number fields)
-  [Click here](${ url_action('display_filtered_data_all')  }) to see all the question tables on one screen.
+
+  Click this link to see [all the question tables on one screen](${ url_action('display_filtered_data_all')  }).
 
   #### Download an excel file:
+
   To extract complete data including text/area/numeric fields, make an excel file:<br>
   [:file-excel: Make Excel Download](${ url_ask('generate_excel') })
 ---
@@ -201,6 +203,7 @@ code: |
       no_reportable_data
 ---
 event: no_interview_stats
+id: interview-has-no-data
 question: Interview has no data!
 subquestion: |  
   The interview you selected is <br>
@@ -211,6 +214,7 @@ buttons:
   - Restart: restart
 ---
 event: no_reportable_data
+id: interview-has-no-reportable-data
 question: Interview has no reportable data, you should use Stats
 subquestion: | 
   The interview you selected is <br>
@@ -244,6 +248,7 @@ code: |
     no_filtered_stats
 ---
 event: no_filtered_stats
+id: no-records-in-this-period
 question: No records in this period
 subquestion: |
   % if date_op == 'between':
@@ -351,14 +356,15 @@ code: |
     xlsx_output.initialize(filename=proper_name + ".xlsx")
     writer = pd.ExcelWriter(xlsx_output.path(),
       engine='xlsxwriter',
-      options={'remove_timezone': True})
+      engine_kwargs={'options': {'remove_timezone': True}})
     excel_formatted_data.to_excel(writer, index=False)
-    writer.save()
+    writer.close()
     xlsx_output.commit()
     del writer
     show_excel_download
 ---
 event: show_excel_download
+id: show-excel-download
 question: |
   Excel Download
 subquestion: |
@@ -374,9 +380,9 @@ code: |
   xlsx_output.initialize(filename=proper_name + ".xlsx")
   writer = pd.ExcelWriter(xlsx_output.path(),
     engine='xlsxwriter',
-    options={'remove_timezone': True})
+    engine_kwargs={'options': {'remove_timezone': True}})
   bg_data.to_excel(writer, index=False)
-  writer.save()
+  writer.close()
   xlsx_output.commit()
   del writer
   background_response_action('excel_resp', the_excel=xlsx_output) 
@@ -390,6 +396,7 @@ subquestion: |
 reload: True
 ---
 event: show_failed_screen
+id: failed-screen
 question: |
   Sorry, something went wrong
 subquestion: |

docassemble_interviewstats-0.6.0/docassemble/InterviewStats/data/questions/stats.yml

@@ -0,0 +1,373 @@
+---
+metadata:
+  title: All Interview Statistics
+  short title: View Stats
+  temporary session: True
+  required privileges:
+    - admin
+    - developer
+    - stats_viewer
+---
+features:
+  javascript:
+    - sortable.js
+  css:
+    - summary.css
+  question help button: True
+  question back button: True
+---
+default screen parts: 
+  footer: |
+    [:sync-alt: Restart](${ url_of('restart') } )
+---
+modules:
+  - .snapshot_statistics
+---
+objects:
+  - xlsx_output: DAFile.using(filename="stats.xlsx")
+---
+#mandatory: True
+code: |
+  message(get_filenames())
+---
+id: interview order
+mandatory: True
+code: |
+  # intro
+  filename
+  simple_data
+  formatted_data
+  display_grouped_data
+  # viewed_filtered  
+---
+event: generate_excel
+code: |
+  if stats_size > 10000:
+    if make_excel_task.ready():
+      if make_excel_task.failed():
+        show_failed_screen
+      show_excel_download
+    else:
+      waiting_screen
+  else:
+    proper_name = space_to_underscore(filename) + "_stats"
+    xlsx_output.initialize(filename=proper_name + ".xlsx")
+    writer = pd.ExcelWriter(xlsx_output.path(),
+      engine='xlsxwriter',
+      engine_kwargs={'options': {'remove_timezone': True}})
+    formatted_data.to_excel(writer, index=False)
+    writer.close()
+    xlsx_output.commit()
+    del writer
+    show_excel_download
+    generate_excel = True
+---
+event: show_excel_download
+id: show-excel-download
+question: |
+  Excel Download
+subquestion: |
+  [:file-excel: Download](${ xlsx_output.url_for() })
+---
+code: |
+  make_excel_task = background_action('excel_task', data=formatted_data)
+---
+event: excel_task
+code: |
+  bg_data = action_argument('data')
+  proper_name = space_to_underscore(filename) + "_stats"
+  xlsx_output.initialize(filename=proper_name + ".xlsx")
+  writer = pd.ExcelWriter(xlsx_output.path(),
+    engine='xlsxwriter',
+    engine_kwargs={'options': {'remove_timezone': True}})
+  bg_data.to_excel(writer, index=False)
+  writer.close()
+  xlsx_output.commit()
+  del writer
+  background_response_action('excel_resp', the_excel=xlsx_output) 
+---
+event: generate_summary_excel
+code: |
+  import pandas as pd
+  proper_name = 'snapshot_summary'
+  xlsx_output.initialize(filename=proper_name + ".xlsx")
+  df = pd.DataFrame(summary_by_filename)
+  writer = pd.ExcelWriter(xlsx_output.path(),
+    engine='xlsxwriter',
+    engine_kwargs={'options': {'remove_timezone': True}})
+  df.to_excel(writer, index=False)
+  writer.close()
+  xlsx_output.commit()
+  del writer
+  show_excel_download
+---
+event: generate_sessions_excel
+code: |
+  import pandas as pd
+  proper_name = 'sessions_summary'
+  xlsx_output.initialize(filename=proper_name + ".xlsx")
+  df = pd.DataFrame(session_summary_by_filename)
+  writer = pd.ExcelWriter(xlsx_output.path(),
+    engine='xlsxwriter',
+    engine_kwargs={'options': {'remove_timezone': True}})
+  df.to_excel(writer, index=False)
+  writer.close()
+  xlsx_output.commit()
+  del writer
+  show_excel_download
+---
+event: waiting_screen
+id: waiting screen
+question: |
+  We're generating your excel sheet.
+subquestion: |
+  This may take a few minutes, so leave this tab open. Thanks for waiting.
+reload: True
+---
+event: show_failed_screen
+id: failed-screen
+question: |
+  Sorry, something went wrong
+subquestion: |
+  We had some trouble trying to assemble your excel sheet.
+
+  ${ make_excel_task.result() }
+buttons:
+  - Restart: restart
+---
+event: excel_resp
+code: |
+  xlsx_output = action_argument('the_excel')
+  background_response()
+---
+code: |
+  summary_stats = get_summary_stats(filename, date_from=stats_date_from if filter_by_date else None, date_to=stats_date_to if filter_by_date else None)
+---
+reconsider: summary_stats
+id: summary-data-screen
+question: |
+  Summary data for ${ interview_title }
+subquestion: |
+  * Total sessions: ${ summary_stats[0] }
+  * First session: ${ format_date(summary_stats[1], format='MM/dd/yyyy') }
+  * Latest session: ${ format_date(summary_stats[2], format='MM/dd/yyyy') }
+  
+  % if summary_stats[0] > 10000:
+  You have a lot of sessions; some operations might
+  take a long time.
+  % endif
+continue button field: simple_data
+---
+# question: Interview Statistics 
+# subquestion: |      
+#   We have two features in this repo:
+  
+#   1. Stats - it generates statistics, maps, charts and a downadable excel file.
+#       - If you have any inteview that uses `store_variables_snapshot(data={'zip': users[0].address.zip})` to save zip data, you should use Stats.
+#       - You can filter your data by date or by state.
+#       - Click the **Continue button** below to use this feature.
+#   1. Data Reporting - it generates statistics tables and a downloadable excel file. 
+#       - If you have any interview that uses `save_input_data()` to save reportable data, you should use Data Reporting.
+#       - You can filter your data by date.
+#       - [Click here to use Data Reporting](${interview_url(i=user_info().package + ":data_report.yml")}) 
+
+# field: intro
+---
+code: |
+  overall_stats = get_overall_stats()
+  summary_by_filename = get_summary_stats_by_filename()
+  session_summary_by_filename = get_session_summary_stats_by_filename()
+---
+id: what-interview-do-you-want-to-view-statistics-for
+question: |
+  What interview do you want to view statistics for?
+under: |
+  #### Summary statistics
+  
+  Keep in mind:
+
+  % if isinstance(get_config("interview delete days"), int):
+  Docassemble sessions on this server are kept for only ${ get_config("interview delete days") } days. 
+  % else:
+  Docassemble sessions typically are kept for only 90 days.
+  % endif
+  Snapshotted sessions
+  may or may not be set to be kept longer. Statistics below will reflect the settings
+  on your server and interviews.
+
+  Total sessions with saved statistics across all interviews: ${ overall_stats[0] }
+
+  % if summary_by_filename:
+  <div style="margin-bottom:0.5rem">
+    <a href="${ url_ask('generate_summary_excel') }" class="btn btn-link">
+      <i class="fa fa-file-excel" aria-hidden="true"></i>
+      Download snapshot summary
+    </a>
+  </div>
+  <h2 class="h4">Snapshotted sessions by interview title</h2>
+  <table id="summary-table" class="table table-striped table-bordered sortable">
+    <thead>
+      <tr>
+        <th>Interview (click to sort)</th>
+        <th data-type="number">30 days</th>
+        <th data-type="number">90 days</th>
+        <th data-type="number">365 days</th>
+        <th data-type="number">All time</th>
+        <th data-type="date">Last (30d)</th>
+        <th data-type="date">Latest</th>
+      </tr>
+    </thead>
+    <tbody>
+    % for row in summary_by_filename:
+      <tr>
+        <td title="${ row['filename'] }"><span class="short-name">${ shorten_filename(row['filename']) }</span></td>
+        <td>${ row.get('count_30d', 0) }</td>
+        <td>${ row.get('count_90d', 0) }</td>
+        <td>${ row.get('count_365d', 0) }</td>
+        <td>${ row.get('count_all', 0) }</td>
+        <td>${ format_date(row.get('max_30d'), format='MM/dd/yyyy') if row.get('max_30d') else '' }</td>
+        <td>${ format_date(row.get('max_all'), format='MM/dd/yyyy') if row.get('max_all') else '' }</td>
+      </tr>
+    % endfor
+    </tbody>
+  </table>
+  % endif
+
+  <p><a href="${ url_ask('session_data_screen') }">View all sessions by interview title</a></p>
+fields:
+  - Filename: filename
+    datatype: combobox
+    code: |
+      get_combined_filename_list()
+  - Filter by date range?: filter_by_date
+    datatype: yesno
+    default: False
+  - From date: stats_date_from
+    datatype: date
+    show if: filter_by_date
+  - To date: stats_date_to
+    datatype: date
+    show if: filter_by_date
+---
+code: |
+  session_summary_by_filename = get_session_summary_stats_by_filename()
+---
+event: session_data_screen
+id: session-data-screen
+question: |
+  All sessions by interview title
+subquestion: |
+  % if session_summary_by_filename:
+  <div style="margin-bottom:0.5rem">
+    <a href="${ url_ask('generate_sessions_excel') }" class="btn btn-link">
+      <i class="fa fa-file-excel" aria-hidden="true"></i>
+      Download session summary
+    </a>
+  </div>
+  <table id="sessions-table" class="table table-striped table-bordered sortable">
+    <thead>
+      <tr>
+          <th>Interview</th>
+          <th data-type="number">30 days</th>
+          <th data-type="number">90 days</th>
+          <th data-type="number">365 days</th>
+          <th data-type="number">All time</th>
+          <th data-type="date">Latest</th>
+        </tr>
+    </thead>
+    <tbody>
+    % for row in session_summary_by_filename:
+      <tr>
+        <td title="${ row['filename'] }"><span class="short-name">${ shorten_filename(row['filename']) }</span></td>
+        <td>${ row.get('count_30d', 0) }</td>
+        <td>${ row.get('count_90d', 0) }</td>
+        <td>${ row.get('count_365d', 0) }</td>
+        <td>${ row.get('count_all', 0) }</td>
+        <td>${ format_date(row.get('max_all'), format='MM/dd/yyyy') if row.get('max_all') else '' }</td>
+      </tr>
+    % endfor
+    </tbody>
+  </table>
+  % else:
+  No session data found.
+  % endif
+
+  [Back to summary](${ url_ask('filename') })
+---
+code: |
+  if filename is not None:
+    if 'data/questions' in filename or 'playground' in filename:
+      stats = get_stats(filename, date_from=stats_date_from if filter_by_date else None, date_to=stats_date_to if filter_by_date else None)
+    else:
+      stats = get_stats(filename.replace(':',':data/questions/'), date_from=stats_date_from if filter_by_date else None, date_to=stats_date_to if filter_by_date else None)
+  else:
+    no_interview_stats
+---
+code: |
+  try:
+    interview_title = next(iter([interview.get('title') for interview in interview_menu() if interview.get('filename') == filename]), filename)
+  except:
+    interview_title = filename
+  # A survey interview's title is saved in Stats, yet interview_title is used on Summary page which comes before we kick out survey interviews, so we set the value here.
+  if stats and 'title' in stats[0].keys():
+    interview_title = stats[0]['title']
+---
+id: there-are-no-interviews-to-look-for-data-in
+event: no_interview_stats
+question: There are no interviews to look for data in!
+subquestion: |
+  There aren't any interviews that we can look for data in. There should be at least one interview
+  listed in the 'dispatch' section of the configuration. See
+  [the docassemble documentation](https://docassemble.org/docs/config.html#dispatch) for more information
+buttons:
+  - Exit: exit
+---
+need: stats
+code: |
+  columns = get_columns(stats)
+  my_col = next(iter(columns), None)
+  groupable_columns = list(set(columns).difference(set(['modtime'])))
+  possible_states = get_column_values(stats, 'state')
+---
+code: |
+  import pandas as pd
+---
+code: |
+  formatted_data = pd.DataFrame()
+  stats_size = 0
+  if stats is not None:
+    formatted_data = pd.DataFrame(stats)
+    stats_size = formatted_data.shape[0]
+---
+event: show_grouped_data
+code: |
+  my_col = action_argument('group_by')
+  force_ask('display_grouped_data')
+---
+id: stats-for
+question: |
+  Stats for ${interview_title}
+subquestion: |
+  Total submissions: ${ len(stats) }
+
+  % if len(stats):
+  Group by:
+  % for col in groupable_columns:
+  [${col}](${ url_action('show_grouped_data', group_by = col)  })
+  % endfor
+
+  <%
+    grouped = formatted_data.groupby(by=my_col).count()
+    grouped = grouped.sort_values(grouped.columns[0], ascending=False)
+    total_groups = len(grouped)
+  %>
+  % if total_groups > 50:
+  Showing the top 50 of ${ total_groups } groups by count. Download the Excel file below for the full breakdown.
+  % endif
+  ${ grouped.head(50).to_html(classes=('table','table-striped')) }
+
+  [:file-excel: Download](${ url_ask('generate_excel') })
+  % else:
+  No records found for this date range. Try a wider range or clear the date filter.
+  % endif
+event: display_grouped_data

{docassemble.InterviewStats-0.5.0 → docassemble_interviewstats-0.6.0}/docassemble/InterviewStats/data/questions/subscribe.yml

@@ -72,6 +72,7 @@ code: |
     formatted_data = pd.DataFrame(stats) 
 ---
 event: no_interview_stats
+id: there-are-no-interviews-to-look-for-data-in
 question: There are no interviews to look for data in!
 subquestion: |
   There aren't any interviews that we can look for data in. There should be at least one interview
@@ -85,9 +86,9 @@ code: |
   xlsx_output.initialize(filename=proper_name + ".xlsx")
   writer = pd.ExcelWriter(xlsx_output.path(),
     engine='xlsxwriter',
-    options={'remove_timezone': True})
+    engine_kwargs={'options': {'remove_timezone': True}})
   formatted_data.to_excel(writer, index=False)
-  writer.save()
+  writer.close()
   xlsx_output.commit()
   del writer
   saved_data = True

{docassemble.InterviewStats-0.5.0 → docassemble_interviewstats-0.6.0}/docassemble/InterviewStats/data/questions/test_create_snapshot_data.yml

@@ -20,12 +20,16 @@ code: |
   if what_to_do == 'generate':
     number_of_data_points
     # Edit your data here
-    ma_zips = ['01001', '01002', '02330', '02332', '02771', '027771',
-      '02125', '02122', '02124', '02123', '02115', '02116']
+    ma_zips = list(set([f'{random.randint(75001, 79999):05}' for _ in range(300)])) \
+        + list(set([f'{random.randint(1001,2799):05}' for _ in range(300)]))
+    import datetime
+    datetimes = [datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) - datetime.timedelta(days=i) for i in range(30)]
     def my_generator():
       for _ in range(number_of_data_points):
         yield {'zip': random.choice(ma_zips), 'state': 'MA'}
-    write_random_user(my_generator())
+    def modtime_generator():
+      return random.choice(datetimes)
+    write_random_user(my_generator(), modtime_generator=modtime_generator)
   elif what_to_do == 'delete':
     which_to_delete
     delete_data_from_interview(which_to_delete)
@@ -36,6 +40,7 @@ code: |
   users[0].name.last = 'Brown'
   users[0].address
 ---
+id: snapshot-generator
 question: |
   Snapshot Generator
 subquestion: |
@@ -49,6 +54,7 @@ fields:
       - Generate: generate
       - Delete: delete
 ---
+id: how-many-data-points-do-you-want-to-generate
 question: |
   How many data points do you want to generate?
 fields:
@@ -58,6 +64,7 @@ fields:
     min: 1
     max: 400000
 ---
+id: what-interview-do-you-want-to-delete-all-statistics-from
 question: |
   What interview do you want to delete all statistics from?
 fields:
@@ -73,6 +80,7 @@ validation code: |
   if which_to_delete_confirm != which_to_delete:
     validation_error("Are you sure you are deleting the right one?")
 ---
+id: snapshot-ending-screen
 event: ending_screen
 question: |
   % if what_to_do == 'generate':

{docassemble.InterviewStats-0.5.0 → docassemble_interviewstats-0.6.0}/docassemble/InterviewStats/data/questions/test_survey_data_generator.yml

@@ -29,6 +29,7 @@ code: |
 #	Question screens
 #--------------------------------------------------------------------------
 ---
+id: survey data generator
 question: |
   Survey Data Generator
 subquestion: |
@@ -42,6 +43,7 @@ fields:
       - Generate: generate
       - Delete: delete
 ---
+id: how-many-data-points-do-you-want-to-generate
 question: |
   How many data points do you want to generate?
 fields:
@@ -52,6 +54,7 @@ fields:
     min: 1
     max: 4000
 ---
+id: which-survey-interviews-do-you-want-to-delete?
 question: |
   Which survey interviews do you want to delete?
 fields:
@@ -67,6 +70,7 @@ validation code: |
   if which_to_delete_confirm != which_to_delete:
     validation_error("Are you sure you are deleting the right one?")
 ---
+id: ending-screen
 event: ending_screen
 question: |
   % if what_to_do == 'generate':