Long-awaited merge of developments with traits back into master. This commit reflects the state of deployment as of 2016-09-15.

Author: tucotuco

app-dev.yaml

@@ -46,6 +46,10 @@ handlers:
 
 - url: /img
   static_dir: www/img
+  
+- url: /favicon\.ico
+  static_files: img/favicon.ico
+  upload: img/favicon\.ico    
 
 # APIs
 

app.yaml

@@ -28,6 +28,13 @@ libraries:
 # - name: webapp2
 #   version: "2.5.1"
 
+# Try the following to see if it overcomes the out of memory issues in the traits version
+# of the app. It didn't.
+#instance_class: B2
+#basic_scaling:
+#  max_instances: 4
+#  idle_timeout: 5m
+
 handlers:
 
 # Static assets
@@ -46,6 +53,10 @@ handlers:
 
 - url: /img
   static_dir: www/img
+  
+- url: /favicon\.ico
+  static_files: img/favicon.ico
+  upload: img/favicon\.ico  
 
 # APIs
 

appengine_config.py

@@ -46,7 +46,12 @@ def namespace_manager_default_namespace_for_request():
         # [dwca-reader-clj "0.10.1-SNAPSHOT"]
 #    	return 'index-2014-02-11a'
     	# DwC2015-style index with traits and days of year
-    	return 'index-2014-02-11'
+#    	return 'index-2014-02-11'
+    	# DwC2015-style index with traits and Atom fields for booleans and some others
+    	return 'index-2013-08-08'
+    	# DwC2015-style small-size test index with lengthtype in addition to traits and 
+    	# Atom fields
+    	#return 'index-2014-02-06t2'
 
 engineauth = {
     # Login uri. The user will be returned here if an error occurs.

portal-web.yaml

@@ -0,0 +1,124 @@
+# This file is part of VertNet: https://github.com/VertNet/webapp
+#
+# VertNet is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# VertNet is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Foobar.  If not, see: http://www.gnu.org/licenses
+
+# This is the App Engine configuration file for local development.
+# Note that static assets are sources from www/ which contains full
+# sourcecode without minification.
+
+# CODEBASE INFO
+# Repository: VertNet/webapp
+# Pushed to branch feature/laura-trait-ui 2016-09-14T23:30_02:00
+# Last pushed to master branch: 
+#
+# DEPLOYMENT INFO
+# Last deployed tuco version: 2016-09-15T20:51+02:00 index-2013-08-08
+# Last deployed pagodarose version: 2016-09-15T20:51+02:00 index-2013-08-08
+# Last deployed prod version: 2016-09-15T20:51+02:00 index-2013-08-08 (from VertNet/webapp repo)
+# command: gcloud app deploy portal-web.yaml
+# args:
+#   --project vertnet-portal  # Optional, no need if gcloud is configured to use this project
+#   --version prod|dev  # Indicates version to use. If 'prod', --promote flag should be used
+#   --promote  # Optional, removes the version id from the deployment URL. Skip for testing and/or dev version
+
+# This is the App Engine configuration file for using gcloud.
+# To deploy for production at http://portal-web.vertnet-portal.appspot.com:
+#   gcloud app deploy portal-web.yaml --version prod --promote
+#
+# Or to deploy for testing at http://tuco.portal-web.vertnet-portal.appspot.com
+#   gcloud app deploy portal-web.yaml --version tuco --no-promote
+
+# Or to deploy for testing at http://pagodarose.vertnet-portal.appspot.com
+#   gcloud app deploy portal-web.yaml --version pagodarose --no-promote
+
+service: default
+runtime: python27
+api_version: 1
+threadsafe: true
+
+libraries:
+- name: jinja2
+  version: "latest"
+# - name: webapp2
+#   version: "2.5.1"
+
+handlers:
+
+# Static assets
+
+- url: /js/lib/*.
+  static_dir: www/lib
+
+- url: /js
+  static_dir: www/js
+
+- url: /css
+  static_dir: www/css
+
+- url: /fonts
+  static_dir: www/fonts
+
+- url: /img
+  static_dir: www/img
+
+# APIs
+
+# - url: /api/search.*
+#   script: vertnet.service.api.handlers
+
+# - url: /api/download.*
+#   script: vertnet.service.api.handlers
+
+- url: /api/user/get
+  script: vertnet.service.user.handler
+
+- url: /api/github.*
+  script: vertnet.service.github.handler
+
+- url: /service/download.*
+  script: vertnet.service.download.api
+
+# - url: /apitracker.*
+#   script: vertnet.service.tracker.api
+
+- url: /service/rpc/record.*
+  script: vertnet.service.record.rpc
+
+- url: /service/organization.*
+  script: vertnet.service.organization.rpc
+
+- url: /api/user.*
+  script: vertnet.service.user.handler
+
+- url: /api/organization.*
+  script: vertnet.api.organization.rpc
+
+# - url: /api/stats.*
+#   script: vertnet.api.stats.rpc
+
+- url: /service/stats.*
+  script: vertnet.service.stats.main
+
+# - url: /service/repochecker.*
+#   script: vertnet.service.repochecker.main
+
+# CRON tasks
+
+# - url: /tasks/daily_portal_stats.*
+#   script: vertnet.service.tasks.daily_portal_stats.main
+
+# Main handler
+
+- url: /.*
+  script: app.handler

queue.yaml

@@ -75,11 +75,11 @@ queue:
 - name: usagestatsqueue
   rate: 35/s
 # index-clean is dangerous - turn it on only if you really need to
-#- name: index-clean
-#  rate: 35/s
-#  retry_parameters:
-#    task_retry_limit: 7
-#    task_age_limit: 60m
-#    min_backoff_seconds: 30
-#    max_backoff_seconds: 960
-#    max_doublings: 7
+- name: index-clean
+  rate: 35/s
+  retry_parameters:
+    task_retry_limit: 7
+    task_age_limit: 60m
+    min_backoff_seconds: 30
+    max_backoff_seconds: 960
+    max_doublings: 7

templates/base.html

@@ -22,7 +22,7 @@
 <html lang="en">
 <head>
     {% block head %}
-    <title>VertNet</title>
+    <title>VertNet Search Portal</title>
     <meta charset="utf-8">
     <meta http-equiv="Accept-Encoding" content="gzip, deflate"/>
     <meta name="viewport" content="width=device-width, initial-scale=1.0">

vertnet/service/download.py

@@ -1,4 +1,21 @@
-"""Download service."""
+#!/usr/bin/env python
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+__author__ = "John Wieczorek"
+__contributors__ = "Aaron Steele, John Wieczorek"
+__copyright__ = "Copyright 2016 vertnet.org"
+__version__ = "download.py 2016-09-14T11:47+02:00"
 
 # Removing dependency on Files API due to its deprecation by Google
 import cloudstorage as gcs
@@ -10,22 +27,23 @@
 from google.appengine.api import taskqueue
 from google.appengine.api import search
 from vertnet.service import util as vnutil
+from vertnet.service.util import UTIL_VERSION
 from vertnet.service import search as vnsearch
 import webapp2
 import json
 import logging
 import uuid
 import sys
+import gc
 
-DOWNLOAD_VERSION='download.py 2015-09-03T10:52:50+02:00'
-
+DOWNLOAD_VERSION=__version__
 SEARCH_CHUNK_SIZE=1000 # limit on documents in a search result: rows per file
 OPTIMUM_CHUNK_SIZE=500 # See api_cnt_performance_analysis.pdf at https://goo.gl/xbLIGz
 COMPOSE_FILE_LIMIT=32 # limit on the number of files in a single compose request
 COMPOSE_OBJECT_LIMIT=1024 # limit on the number of files in a composition
 TEMP_BUCKET='vn-dltest' # bucket for temp compositions
 DOWNLOAD_BUCKET='vn-downloads2' # production bucket for downloads
-FILE_EXTENSION='tsv'
+FILE_EXTENSION='txt'
 
 def _tsv(json):
     # These should be the names of the original fields in the index document.
@@ -40,6 +58,9 @@ def _tsv(json):
                 values.append(unicode(json[x]).rstrip())        
         else:
             values.append(u'')
+#     logging.debug('%s: JSON: %s' % (DOWNLOAD_VERSION, json))
+#     logging.debug('%s: DOWNLOAD_FIELDS: %s' % (UTIL_VERSION, download_fields))
+#     logging.debug('%s: VALUES: %s' % (DOWNLOAD_VERSION, values))
     return u'\t'.join(values).encode('utf-8')
 
 def _get_tsv_chunk(records):
@@ -92,6 +113,8 @@ def _queue(self, q, email, name, latlon, fromapi, source, countonly):
             latlon=latlon, fileindex=0, reccount=0, requesttime=requesttime, 
             source=source, fromapi=fromapi)
 
+        # Attempt to keep memory usage at a minimum
+        gc.collect()
         if countonly is not None and len(countonly)>0:
             taskqueue.add(url='/service/download/count', params=params, 
                 queue_name="count")
@@ -159,6 +182,7 @@ def get(self):
             self.response.headers['Content-Disposition'] = "attachment; filename=%s" \
                 % filename
             records, cursor, count, query_version = vnsearch.query(q, count)
+            # logging.debug('%s: RECORDS: %s' % (DOWNLOAD_VERSION, records))
 
             # Build dictionary for search counts
             res_counts = vnutil.search_resource_counts(records)
@@ -210,6 +234,8 @@ def post(self):
         cursor = self.request.get('cursor')
         email = self.request.get('email')
 
+        # Attempt to keep memory usage at a minimum
+        gc.collect()
         if cursor:
             curs = search.Cursor(web_safe_string=cursor)
         else:
@@ -279,6 +305,8 @@ def post(self):
         else:
             curs = None
 
+        # Attempt to keep memory usage at a minimum
+        gc.collect()
         # Write single chunk to file, GCS does not support append
         records, next_cursor, count, query_version = \
             vnsearch.query(q, SEARCH_CHUNK_SIZE, curs=curs)
@@ -317,9 +345,9 @@ def post(self):
                         f.write('%s\n' % vnutil.download_header())
                     f.write(chunk)
                     success = True
-                    logging.info('Download chunk saved to %s: Total %s records. Has next \
-cursor: %s \nVersion: %s' 
-                        % (filename, reccount, not next_cursor is None, DOWNLOAD_VERSION))
+#                    logging.info('Download chunk saved to %s: Total %s records. Has next \
+#cursor: %s \nVersion: %s' 
+#                        % (filename, reccount, not next_cursor is None, DOWNLOAD_VERSION))
             except Exception, e:
                 logging.error("Error writing chunk to FILE: %s for\nQUERY: %s \
 Error: %s\nVersion: %s" % (filename, q, e, DOWNLOAD_VERSION) )
@@ -404,6 +432,8 @@ def post(self):
         total_files_to_compose = int(self.request.get('fileindex'))+1
         compositions=total_files_to_compose
 
+        # Attempt to keep memory usage at a minimum
+        gc.collect()
         # Get the application default credentials.
         credentials = GoogleCredentials.get_application_default()
 
@@ -499,8 +529,9 @@ def post(self):
         try:
             gcs.copy2(src, dest)
         except Exception, e:
-            logging.error("Error copying %s to %s \nError: %s\
-Version: %s" % (src, dest, e, DOWNLOAD_VERSION) )
+            s = 'Error copying %s to %s\n' % (src, dest)
+            s += 'Error: %s Version: %s' % (e, DOWNLOAD_VERSION) 
+            logging.error()
 
         # Change the ACL so that the download file is publicly readable.
         mbody=acl_update_request()
@@ -519,7 +550,7 @@ def post(self):
             mail.send_mail(sender="VertNet Downloads <vertnetinfo@vertnet.org>", 
                 to=email, subject="Your truncated VertNet download is ready!",
                 body="""
-Your VertNet download file is now available for a limited time at 
+Your VertNet download file is now available for a limited time (roughly 60 days) at 
 https://storage.googleapis.com/%s/%s.\n
 The results in this file are not complete based on your query\n
 %s\n
@@ -537,7 +568,7 @@ def post(self):
             mail.send_mail(sender="VertNet Downloads <vertnetinfo@vertnet.org>", 
                 to=email, subject="Your VertNet download is ready!",
                 body="""
-Your VertNet download file is now available for a limited time at 
+Your VertNet download file is now available for a limited time (roughly 60 days) at 
 https://storage.googleapis.com/%s/%s.\n
 Query: %s\nMatching records: %s\nRequest submitted: %s\nRequest fulfilled: %s""" 
                     % (DOWNLOAD_BUCKET, composed_filename, q, reccount, requesttime, 
@@ -559,6 +590,8 @@ def post(self):
         composed_filename='%s.%s' % (filepattern,FILE_EXTENSION)
         total_files_to_compose = int(self.request.get('fileindex'))
 
+        # Attempt to keep memory usage at a minimum
+        gc.collect()
         # Get the application default credentials.
         credentials = GoogleCredentials.get_application_default()