Convert to python 3

.travis.yml

@@ -1,6 +1,6 @@
 language: python
 python:
-  - "2.7"
+  - "3.5"
 env:
   - REFL_INSTALL_DIR=/tmp
 

Makefile

@@ -5,7 +5,6 @@ else
 endif
 
 app_dir := web_reflectivity
-DJANGO_COMPATIBLE:=$(shell python -c "import django;t=0 if django.VERSION[1]<9 else 1; print t")
 DJANGO_VERSION:=$(shell python -c "import django;print django.__version__")
 
 ifndef PYTHON
@@ -23,12 +22,7 @@ check:
 	@python -c "import pandas" || echo "\nWARNING: pandas is not installed\n"
 	@python -c "import plotly" || echo "\nWARNING: plotly is not installed\n"
 	@python -c "import plotly.offline" || echo "\nWARNING: plotly.offline is not installed\n"
-
-ifeq ($(DJANGO_COMPATIBLE),1)
 	@echo "Detected Django $(DJANGO_VERSION)"
-else
-	$(error Detected Django $(DJANGO_VERSION) < 1.9. The web monitor requires at least Django 1.9)
-endif
 
 deps:
 	$(PYTHON) -m pip install -r requirements.txt

test/environment/webrefl_env_py3.yml

@@ -0,0 +1,19 @@
+name: webrefl_py3
+dependencies:
+- python==3.5
+- numpy
+- sqlite
+- pip:
+  - django
+  - celery
+  - django-celery-results
+  - django-remote-submission
+  - cython
+  - pandas
+  - plotly
+  - scipy
+  - requests
+  - periodictable
+  - refl1d
+  - matplotlib
+

web_reflectivity/datahandler/models.py

@@ -23,7 +23,7 @@ class DataRun(models.Model):
     # Optional free-form run identifier
     run_id = models.TextField()
 
-    instrument = models.ForeignKey(Instrument)
+    instrument = models.ForeignKey(Instrument, models.CASCADE)
     created_on = models.DateTimeField('Timestamp', auto_now_add=True)
 
     def __unicode__(self):
@@ -35,7 +35,7 @@ class PlotData(models.Model):
         Table of plot data. This data can either be json or html
     """
     ## DataRun this run status belongs to
-    data_run = models.ForeignKey(DataRun)
+    data_run = models.ForeignKey(DataRun, models.CASCADE)
 
     ## JSON/HTML data
     data = models.TextField()

web_reflectivity/fitting/__init__.py

@@ -1,5 +1,3 @@
-from __future__ import absolute_import, unicode_literals
-
 # This will make sure the app is always imported when
 # Django starts so that shared_task will use this app.
 from .celery import app as celery_app

web_reflectivity/fitting/apps.py

@@ -1,5 +1,3 @@
-from __future__ import unicode_literals
-
 from django.apps import AppConfig
 
 

web_reflectivity/fitting/catalog.py

@@ -35,7 +35,7 @@ def decode_time(timestamp):
                 date_time_str = date_time_str[:sec_location]
             return datetime.datetime.strptime(date_time_str, "%Y-%m-%dT%H:%M:%S")
     except:
-        logging.error("Could not parse timestamp '%s': %s", timestamp, sys.exc_value)
+        logging.error("Could not parse timestamp '%s': %s", timestamp, sys.exc_info()[1])
         return None
 
 def get_run_info(instrument, run_number):
@@ -93,6 +93,6 @@ def _get_run_info(instrument, run_number, facility='SNS'):
             run_info['proposal'] = datafiles[0].experiment
             run_info['location'] = datafiles[0].location
     except:
-        logging.error("Communication with ONCat server failed: %s", sys.exc_value)
+        logging.error("Communication with ONCat server failed: %s", sys.exc_info()[1])
 
     return run_info

web_reflectivity/fitting/celery.py

@@ -1,4 +1,3 @@
-from __future__ import absolute_import, unicode_literals
 import os
 from celery import Celery
 

web_reflectivity/fitting/data_server/data_handler.py

@@ -3,11 +3,10 @@
     Abstraction of the data handler. For test purposes, we can store data locally.
     With a production system, we are likely to set the data to a remote server.
 """
-from __future__ import absolute_import, division, print_function, unicode_literals
 import sys
 import logging
 import json
-import httplib
+import http.client as httplib
 import hashlib
 import string
 import requests
@@ -192,7 +191,7 @@ def _remote_fetch(instrument, run_id, data_type='html'):
         else:
             logging.error("Return code %s for %s:", data_request.status, live_data_url)
     except:
-        logging.error("Could not pull data from live data server:\n%s", sys.exc_value)
+        logging.error("Could not pull data from live data server:\n%s", sys.exc_info()[1])
     return json_data
 
 def get_user_files_from_server(request, filter_file_name=None):
@@ -225,4 +224,4 @@ def get_user_files_from_server(request, filter_file_name=None):
                 user_data.timestamp = dateparse.parse_datetime(item['timestamp'])
                 user_data.save()
     except:
-        logging.error("Could not retrieve user files: %s", sys.exc_value)
+        logging.error("Could not retrieve user files: %s", sys.exc_info()[1])

web_reflectivity/fitting/icat_server_communication.py

@@ -7,7 +7,7 @@
     @copyright: 2015 Oak Ridge National Laboratory
 """
 import sys
-import httplib
+import http.client as httplib
 import xml.dom.minidom
 import logging
 from fitting.models import CatalogCache
@@ -17,7 +17,7 @@
     ICAT_DOMAIN = settings.ICAT_DOMAIN
     ICAT_PORT = settings.ICAT_PORT
 except:
-    logging.warning("Could not find ICAT config: %s", sys.exc_value)
+    logging.warning("Could not find ICAT config: %s", sys.exc_info()[1])
     ICAT_DOMAIN = 'icat.sns.gov'
     ICAT_PORT = 2080
 
@@ -66,6 +66,6 @@ def get_run_info(instrument, run_number):
                                                title=run_info['title'],
                                                proposal=run_info['proposal'])
     except:
-        logging.error("Communication with ICAT server failed (%s): %s",url, sys.exc_value)
+        logging.error("Communication with ICAT server failed (%s): %s",url, sys.exc_info()[1])
 
     return run_info

web_reflectivity/fitting/models.py

@@ -3,7 +3,6 @@
     Data models
 """
 #TODO: move the script generation from the forms to the models
-from __future__ import unicode_literals
 import sys
 import logging
 import re
@@ -248,12 +247,12 @@ def apply_constraint(self, fit_problem):
             def constraint_func():
                 """ Dummy function to avoid pylint error """
                 return 1
-            exec constraint_function #pylint: disable=exec-used
+            exec(constraint_function) #pylint: disable=exec-used
             setattr(self.layer, self.parameter, constraint_func(**parameters))
             self.layer.save()
             fit_problem.save()
         except:
-            logging.error("Could not evaluate constraint: %s", sys.exc_value)
+            logging.error("Could not evaluate constraint: %s", sys.exc_info()[1])
 
     def get_ranges(self, sample_name='sample', probe_name='probe'):
         """
@@ -313,9 +312,9 @@ def constraint():
                 return 1
 
             compile(constraint_function, 'constraint.py', 'exec')
-            exec constraint_function #pylint: disable=exec-used
+            exec(constraint_function) #pylint: disable=exec-used
         except:
-            comments.append("Syntax error:\n%s" % sys.exc_value)
+            comments.append("Syntax error:\n%s" % sys.exc_info()[1])
             is_valid = False
 
         try:
@@ -332,7 +331,7 @@ def constraint():
                 is_valid = False
         except:
             comments.append("Invalid parameters. Check your function.")
-            comments.append("Syntax error:\n%s" % sys.exc_value)
+            comments.append("Syntax error:\n%s" % sys.exc_info()[1])
             is_valid = False
         return is_valid, comments
 
@@ -445,7 +444,7 @@ def _retrieve_info(self, obj_id, par_name):
                 problem_id = fit_problem.id
                 dependent_name = layer.name
             except:
-                logging.error(sys.exc_value)
+                logging.error(sys.exc_info()[1])
                 logging.error("Could not retrieve layer id=%s", obj_id)
         return dependent_name, parameter_name, problem_id
 

web_reflectivity/fitting/parsing/refl1d_err_model.py

@@ -107,7 +107,7 @@ def translate_model(refl_model, layers, output_params=None):
                 clean_layers.insert(0, layer_dict)
         except:
             logging.error("Could not process %s", str(layers))
-            logging.error(sys.exc_value)
+            logging.error(sys.exc_info()[1])
     return [refl_model, clean_layers]
 
 def parse_slabs(content):

web_reflectivity/fitting/simultaneous/model_handling.py

@@ -50,7 +50,7 @@ def get_simultaneous_models(request, fit_problem, setup_request=False):
                 else:
                     error_list.append("No results found")
         except:
-            logging.error("Problem retrieving results: %s", sys.exc_value)
+            logging.error("Problem retrieving results: %s", sys.exc_info()[1])
             error_list.append("Problem retrieving results")
             raise
 
@@ -194,7 +194,7 @@ def create_plots_from_legacy_log(request, fit_problem):
                     sld_names.append(data_path)
         except:
             logging.error("Could not extract data from log for %s", fit_problem.reflectivity_model.data_path)
-            logging.error(sys.exc_value)
+            logging.error(sys.exc_info()[1])
     return data_list, data_names, sld_list, sld_names
 # --------------------------------------------------------------------------------------
 

web_reflectivity/fitting/urls.py

@@ -6,6 +6,7 @@
 
 from . import views
 
+app_name = 'fitting'
 urlpatterns = [
     url(r'^$',                                                    views.FileView.as_view(),       name='modeling'),
     url(r'^(?P<job_id>\d+)/$',                                    views.is_completed,             name='is_completed'),

web_reflectivity/fitting/view_util.py

@@ -2,7 +2,6 @@
 """
     Utilities for modeling application
 """
-from __future__ import absolute_import, division, print_function, unicode_literals
 import sys
 import os
 import re
@@ -21,7 +20,7 @@
 from django.utils import dateformat, timezone
 from django_remote_submission.models import Server, Job, Log, Interpreter
 from django_remote_submission.tasks import submit_job_to_server, LogPolicy
-from django.core.urlresolvers import reverse
+from django.urls import reverse
 from django.http import Http404
 
 import users.view_util
@@ -68,7 +67,7 @@ def extract_ascii_from_div(html_data):
                 return ascii_data
     except:
         # Unable to extract data from <div>
-        logging.debug("Unable to extract data from <div>: %s", sys.exc_value)
+        logging.debug("Unable to extract data from <div>: %s", sys.exc_info()[1])
     return None
 
 def check_permissions(request, run_id, instrument):
@@ -197,7 +196,7 @@ def get_results(request, fit_problem):
                 else:
                     errors.append("No results found")
             except:
-                logging.error("Problem retrieving results: %s", sys.exc_value)
+                logging.error("Problem retrieving results: %s", sys.exc_info()[1])
                 errors.append("Problem retrieving results")
 
         # Regardless of whether we have a fit result, we can still show the model.
@@ -342,8 +341,8 @@ def evaluate_model(data_form, layers_form, html_data, fit=True, user=None, run_i
         return _evaluate_model(data_form, layers_form, html_data, fit=fit, user=user, run_info=run_info)
     except:
         traceback.print_exc()
-        logging.error("Problem evaluating model: %s", sys.exc_value)
-        return {'error': "Problem evaluating model: %s" % sys.exc_value}
+        logging.error("Problem evaluating model: %s", sys.exc_info()[1])
+        return {'error': "Problem evaluating model: %s" % sys.exc_info()[1]}
 
 def _evaluate_model(data_form, layers_form, html_data, fit=True, user=None, run_info=None):
     """
@@ -737,7 +736,7 @@ def parse_ascii_file(request, file_name, raw_content):
     """
     try:
         # If we don't have a fourth column, add 3% Q resolution
-        current_str = io.StringIO(unicode(raw_content))
+        current_str = io.StringIO(raw_content.decode("utf-8"))
         current_data = pandas.read_csv(current_str, delim_whitespace=True, comment='#', names=['q','r','dr','dq'])
         data_set = [current_data['q'], current_data['r'], current_data['dr'], current_data['dq']]
         for i in range(len(current_data['dq'])):
@@ -750,7 +749,9 @@ def parse_ascii_file(request, file_name, raw_content):
         # Upload plot to live data server
         return data_handler.store_user_data(request, file_name, plot)
     except:
-        logging.error("Could not parse file %s: %s", file_name, sys.exc_value)
+        err_info = sys.exc_info()
+        logging.error("Could not parse file %s: %s", file_name, err_info[1])
+        logging.error('\n'.join(traceback.format_tb(err_info[2])))
         return False, "Could not parse data file %s" % file_name
 
 def get_user_files(request):

web_reflectivity/fitting/views.py

@@ -2,13 +2,12 @@
 """
     Definition of views
 """
-from __future__ import absolute_import, division, print_function, unicode_literals
 import sys
 import logging
 import json
 from django.shortcuts import render, redirect, get_object_or_404
 from django.forms.formsets import formset_factory
-from django.core.urlresolvers import reverse
+from django.urls import reverse
 from django.conf import settings
 from django.contrib.auth.decorators import login_required
 from django.http import HttpResponse, HttpResponseNotFound, Http404
@@ -117,6 +116,9 @@ def post(self, request, *args, **kwargs):
         if form.is_valid():
             file_name = request.FILES['file'].name
             if request.FILES['file'].size < 1024 * 1024:
+                #raw_content = ''
+                #for chunk in request.FILES['file'].chunks():
+                #    logging.error("TYPE: %s", type(chunk))
                 raw_content = request.FILES['file'].read()
                 success, error_msg = view_util.parse_ascii_file(request, file_name, raw_content)
                 if success is True:
@@ -425,7 +427,7 @@ def post(self, request, instrument, data_id, *args, **kwargs):
             else:
                 error_message.append("Invalid parameters were found!")
         except:
-            logging.error("Could not fit data: %s", sys.exc_value)
+            logging.error("Could not fit data: %s", sys.exc_info()[1])
             error_message.append("Could not fit data")
 
         # If we have errors, compose the response here so that we can display the errors.
@@ -708,7 +710,7 @@ def post(self, request, instrument, data_id, *args, **kwargs):
             if 'job_id' in output:
                 request.session['job_id'] = output['job_id']
         except:
-            error_list = ["There was a problem performing your fit:<br>refresh your page.", str(sys.exc_value)]
+            error_list = ["There was a problem performing your fit:<br>refresh your page.", str(sys.exc_info()[1])]
         if len(error_list) > 0:
             breadcrumbs = "<a href='/'>home</a> &rsaquo; simultaneous &rsaquo; %s &rsaquo; %s" % (instrument, data_id)
             template_values = dict(breadcrumbs=breadcrumbs, instrument=instrument,

web_reflectivity/tools/views.py

@@ -39,8 +39,8 @@ def post(self, request, *args, **kwargs):
                 else:
                     template_values.update(electrode_info)
             except:
-                logging.error(sys.exc_value)
-                template_values['user_alert']= ["Could not process request.", "Check your density value.", sys.exc_value]
+                logging.error(sys.exc_info()[1])
+                template_values['user_alert']= ["Could not process request.", "Check your density value.", sys.exc_info()[1]]
         else:
             template_values['user_alert']= ["The form contains invalid data"]
 

web_reflectivity/users/urls.py

@@ -2,10 +2,11 @@
 """
     Define url structure
 """
-from django.conf.urls import url
+from django.urls import include, path
 from . import views
 
+app_name = 'users'
 urlpatterns = [
-    url(r'^login$', views.perform_login, name='perform_login'),
-    url(r'^logout$', views.perform_logout, name='perform_logout'),
+    path('login/', views.perform_login, name='perform_login'),
+    path('logout/', views.perform_logout, name='perform_logout'),
 ]