[launchpad] Add support for launchpad

README.md

@@ -56,6 +56,7 @@ Each enriched index includes one or more types of documents, which are summarize
 - **Jenkins**: each document corresponds to a single built.
 - **Jira**: each document corresponds to an issue or a comment. To simplify counting user activities, issues are duplicated and they can include assignee, reporter and creator data respectively.
 - **Kitsune**: each document can be either a question or an answer.
+- **Launchpad**: each document corresponds to a bug.
 - **Mattermost**: each document corresponds to a message.
 - **Mbox**: each document corresponds to a message.
 - **Mediawiki**: each document corresponds to a review.

grimoire_elk/enriched/launchpad.py

@@ -0,0 +1,203 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2015-2020 Bitergia
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+# Authors:
+#   Nitish Gupta <imnitish.ng@gmail.com>
+#
+
+import logging
+
+from grimoirelab_toolkit.datetime import (datetime_utcnow)
+
+from .utils import get_time_diff_days
+
+from .enrich import Enrich, metadata
+from ..elastic_mapping import Mapping as BaseMapping
+
+logger = logging.getLogger(__name__)
+
+
+class Mapping(BaseMapping):
+
+    @staticmethod
+    def get_elastic_mappings(es_major):
+        """Get Elasticsearch mapping.
+
+        geopoints type is not created in dynamic mapping
+
+        :param es_major: major version of Elasticsearch, as string
+        :returns:        dictionary with a key, 'items', with the mapping
+        """
+
+        mapping = """
+        {
+            "properties": {
+               "description_analyzed": {
+                    "type": "text",
+                    "index": true
+               }
+           }
+        }
+        """
+        return {"items": mapping}
+
+
+class LaunchpadEnrich(Enrich):
+
+    mapping = Mapping
+    roles = ['assignee_data', 'owner_data']
+
+    def __init__(self, db_sortinghat=None, db_projects_map=None, json_projects_map=None,
+                 db_user='', db_password='', db_host=''):
+        super().__init__(db_sortinghat, db_projects_map, json_projects_map,
+                         db_user, db_password, db_host)
+
+    def get_field_author(self):
+        return "owner_data"
+
+    def set_elastic(self, elastic):
+        self.elastic = elastic
+
+    def get_sh_identity(self, item, identity_field=None):
+        """ Return a Sorting Hat identity using launchpad user data """
+
+        identity = {
+            'username': None,
+            'name': None,
+            'email': None
+        }
+
+        item = item['data']
+        if isinstance(item, dict) and identity_field in item:
+            identity['username'] = item[identity_field].get('name', None)
+            identity['name'] = item[identity_field].get('display_name', None)
+
+        return identity
+
+    def get_identities(self, item):
+        """ Return the identities from an item """
+
+        for rol in self.roles:
+            if rol in item['data']:
+                user = self.get_sh_identity(item, rol)
+                yield user
+
+    @metadata
+    def get_rich_item(self, item):
+
+        eitem = {}
+
+        self.copy_raw_fields(self.RAW_FIELDS_COPY, item, eitem)
+
+        # data fields to copy
+        copy_fields = ["title", "web_link", "date_created", "date_incomplete", "is_complete",
+                       "status", "bug_target_name", "importance", "date_triaged", "date_left_new"]
+
+        bug = item['data']
+        self.copy_raw_fields(copy_fields, bug, eitem)
+
+        if self.sortinghat:
+            eitem.update(self.get_item_sh(item, self.roles))
+
+        if self.prjs_map:
+            eitem.update(self.get_item_project(eitem))
+
+        eitem.update(self.__get_rich_bugs(bug))
+        eitem.update(self.get_grimoire_fields(bug["date_created"], "issue"))
+
+        return eitem
+
+    def __get_rich_bugs(self, data):
+        """Create enriched data for bugs"""
+
+        rich_bugtask = {}
+
+        # Time to
+        if not data["is_complete"]:
+            rich_bugtask["time_open_days"] = get_time_diff_days(data['date_created'], datetime_utcnow().replace(tzinfo=None))
+        else:
+            rich_bugtask["time_open_days"] = get_time_diff_days(data['date_created'], data['date_closed'])
+            rich_bugtask["time_created_to_assigned"] = get_time_diff_days(data['date_created'], data['date_assigned'])
+            rich_bugtask['time_assigned_to_closed'] = get_time_diff_days(data['date_assigned'], data['date_closed'])
+            rich_bugtask["time_to_close_days"] = get_time_diff_days(data['date_created'], data['date_closed'])
+
+        if data['activity_data']:
+            rich_bugtask['time_to_last_update_days'] = \
+                get_time_diff_days(data['date_created'], data['activity_data'][-1]['datechanged'])
+
+        rich_bugtask['reopened'] = 1 if data['date_left_closed'] else 0
+        rich_bugtask['time_to_fix_commit'] = get_time_diff_days(data['date_created'], data['date_fix_committed'])
+        rich_bugtask['time_worked_on'] = get_time_diff_days(data['date_in_progress'], data['date_fix_committed'])
+        rich_bugtask['time_to_confirm'] = get_time_diff_days(data['date_created'], data['date_confirmed'])
+
+        # Author and assignee data
+        owner = data.get('owner_data', None)
+        if owner:
+            rich_bugtask['user_login'] = owner.get('name', None)
+            rich_bugtask['user_name'] = owner.get('display_name', None)
+            rich_bugtask['user_joined'] = owner.get('date_created', None)
+            rich_bugtask['user_karma'] = owner.get('karma', None)
+            rich_bugtask['user_time_zone'] = owner.get('time_zone', None)
+
+        assignee = data.get('assignee_data', None)
+        if assignee:
+            assignee = data['assignee_data']
+            rich_bugtask['assignee_login'] = assignee.get('name', None)
+            rich_bugtask['assignee_name'] = assignee.get('display_name', None)
+            rich_bugtask['assignee_joined'] = assignee.get('date_created', None)
+            rich_bugtask['assignee_karma'] = assignee.get('karma', None)
+            rich_bugtask['assignee_time_zone'] = assignee.get('time_zone', None)
+
+        # Extract info related to bug
+        rich_bugtask.update(self.__extract_bug_info(data['bug_data']))
+
+        rich_bugtask['time_to_first_attention'] = \
+            get_time_diff_days(data['date_created'], self.get_time_to_first_attention(data))
+        rich_bugtask['activity_count'] = len(data['activity_data'])
+
+        return rich_bugtask
+
+    def __extract_bug_info(self, bug_data):
+
+        rich_bug_info = {}
+
+        copy_fields = ["latest_patch_uploaded", "security_related", "private", "users_affected_count",
+                       "title", "description", "tags", "date_last_updated", "message_count", "heat"]
+        rich_bug_info['time_created_to_last_update_days'] = \
+            get_time_diff_days(bug_data['date_created'], bug_data['date_last_updated'])
+
+        rich_bug_info['description'] = bug_data['description'][:self.KEYWORD_MAX_LENGTH]
+        rich_bug_info['description_analyzed'] = bug_data['description']
+        rich_bug_info['bug_name'] = bug_data['name']
+        rich_bug_info['bug_id'] = bug_data['id']
+
+        self.copy_raw_fields(copy_fields, bug_data, rich_bug_info)
+
+        return rich_bug_info
+
+    def get_time_to_first_attention(self, item):
+        """Get the first date at which a comment or activity was made to the bug by someone
+        other than the user who created the issue
+        """
+        message_dates = [message['date_created'] for message in item['messages_data']
+                         if item['owner_data'].get('name', None) != message['owner_data'].get('name', None)]
+        activity_dates = [activity['datechanged'] for activity in item['activity_data']
+                          if item['owner_data'].get('name', None) != activity['person_data'].get('name', None)]
+        activity_dates.extend(message_dates)
+        if activity_dates:
+            return min(activity_dates)
+        return None

grimoire_elk/raw/launchpad.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# Copyright (C) 2015-2019 Bitergia
+# Copyright (C) 2015-2020 Bitergia
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -16,13 +16,52 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 # Authors:
-#   Valerio Cosentino <valcos@bitergia.com>
+#   Nitish Gupta <imnitish.ng@gmail.com>
 #
 
 from .elastic import ElasticOcean
+from ..elastic_mapping import Mapping as BaseMapping
+
+
+class Mapping(BaseMapping):
+
+    @staticmethod
+    def get_elastic_mappings(es_major):
+        """Get Elasticsearch mapping.
+
+        :param es_major: major version of Elasticsearch, as string
+        :returns:        dictionary with a key, 'items', with the mapping
+        """
+
+        mapping = '''
+             {
+                "dynamic":true,
+                "properties": {
+                    "data": {
+                        "dynamic":false,
+                        "properties": {}
+                    }
+                }
+            }
+            '''
+
+        return {"items": mapping}
 
 
 class LaunchpadOcean(ElasticOcean):
     """Launchpad Ocean feeder"""
 
-    pass
+    mapping = Mapping
+
+    @classmethod
+    def get_perceval_params_from_url(cls, url):
+        params = []
+        splits = url.split('/')
+        if '+source' in url:
+            params.append(splits[-3])
+            params.append('--package')
+            params.append(splits[-1])
+        else:
+            params.append(splits[-1])
+
+        return params

grimoire_elk/utils.py

@@ -53,6 +53,7 @@
 from perceval.backends.core.hyperkitty import HyperKitty, HyperKittyCommand
 from perceval.backends.core.jenkins import Jenkins, JenkinsCommand
 from perceval.backends.core.jira import Jira, JiraCommand
+from perceval.backends.core.launchpad import Launchpad, LaunchpadCommand
 from perceval.backends.core.mattermost import Mattermost, MattermostCommand
 from perceval.backends.core.mbox import MBox, MBoxCommand
 from perceval.backends.core.mediawiki import MediaWiki, MediaWikiCommand
@@ -101,6 +102,7 @@
 from .enriched.jenkins import JenkinsEnrich
 from .enriched.jira import JiraEnrich
 from .enriched.kitsune import KitsuneEnrich
+from .enriched.launchpad import LaunchpadEnrich
 from .enriched.mattermost import MattermostEnrich
 from .enriched.mbox import MBoxEnrich
 from .enriched.mediawiki import MediaWikiEnrich
@@ -142,6 +144,7 @@
 from .raw.jenkins import JenkinsOcean
 from .raw.jira import JiraOcean
 from .raw.kitsune import KitsuneOcean
+from .raw.launchpad import LaunchpadOcean
 from .raw.mattermost import MattermostOcean
 from .raw.mbox import MBoxOcean
 from .raw.mediawiki import MediaWikiOcean
@@ -242,6 +245,7 @@ def get_connectors():
             "jenkins": [Jenkins, JenkinsOcean, JenkinsEnrich, JenkinsCommand],
             "jira": [Jira, JiraOcean, JiraEnrich, JiraCommand],
             "kitsune": [Kitsune, KitsuneOcean, KitsuneEnrich, KitsuneCommand],
+            "launchpad": [Launchpad, LaunchpadOcean, LaunchpadEnrich, LaunchpadCommand],
             "mattermost": [Mattermost, MattermostOcean, MattermostEnrich, MattermostCommand],
             "mbox": [MBox, MBoxOcean, MBoxEnrich, MBoxCommand],
             "mediawiki": [MediaWiki, MediaWikiOcean, MediaWikiEnrich, MediaWikiCommand],

releases/unreleased/add-support-for-launchpad.yml

@@ -0,0 +1,8 @@
+---
+title: Add support for Launchpad
+category: added
+author: Nitish Gupta <imnitish.ng@gmail.com>
+issue: 842
+notes: Added support for creating raw and enriched
+       indexes of launchpad bugs. The schemas and 
+       tests for data have also been added.