feat: Add support for tracking Data Issues via Asana (#700)

Signed-off-by: Nathan Lawrence <nathanlawrence@asana.com>

amundsen_application/config.py

@@ -64,6 +64,7 @@ class Config:
 
     # Settings for Issue tracker integration
     ISSUE_LABELS = []  # type: List[str]
+    ISSUE_TRACKER_API_TOKEN = None  # type: str
     ISSUE_TRACKER_URL = None  # type: str
     ISSUE_TRACKER_USER = None  # type: str
     ISSUE_TRACKER_PASSWORD = None  # type: str

amundsen_application/models/data_issue.py

 # Copyright Contributors to the Amundsen project.
 # SPDX-License-Identifier: Apache-2.0
 
-# JIRA SDK does not return priority beyond the name
-PRIORITY_MAP = {
-    'Blocker': 'P0',
-    'Critical': 'P1',
-    'Major': 'P2',
-    'Minor': 'P3'
-}
+from enum import Enum
+from typing import Optional
+
+
+class Priority(Enum):
+    P0 = ('P0', 'Blocker')
+    P1 = ('P1', 'Critical')
+    P2 = ('P2', 'Major')
+    P3 = ('P3', 'Minor')
+
+    def __init__(self, level: str, jira_severity: str):
+        self.level = level
+        self.jira_severity = jira_severity
+
+    # JIRA SDK does not return priority beyond the name
+    @staticmethod
+    def from_jira_severity(jira_severity: str) -> 'Optional[Priority]':
+        jira_severity_to_priority = {
+            p.jira_severity: p for p in Priority
+        }
+
+        return jira_severity_to_priority.get(jira_severity)
+
+    @staticmethod
+    def from_level(level: str) -> 'Optional[Priority]':
+        level_to_priority = {
+            p.level: p for p in Priority
+        }
+
+        return level_to_priority.get(level)
 
 
 class DataIssue:
@@ -16,22 +39,17 @@ class DataIssue:
                  title: str,
                  url: str,
                  status: str,
-                 priority: str) -> None:
+                 priority: Optional[Priority]) -> None:
         self.issue_key = issue_key
         self.title = title
         self.url = url
         self.status = status
-        if priority in PRIORITY_MAP:
-            self.priority_display_name = PRIORITY_MAP[priority]
-            self.priority_name = priority.lower()
-        else:
-            self.priority_display_name = None  # type: ignore
-            self.priority_name = None  # type: ignore
+        self.priority = priority
 
     def serialize(self) -> dict:
         return {'issue_key': self.issue_key,
                 'title': self.title,
                 'url': self.url,
                 'status': self.status,
-                'priority_name': self.priority_name,
-                'priority_display_name': self.priority_display_name}
+                'priority_name': self.priority.jira_severity.lower() if self.priority else None,
+                'priority_display_name': self.priority.level if self.priority else None}

amundsen_application/proxy/issue_tracker_clients/asana_client.py

+# Copyright Contributors to the Amundsen project.
+# SPDX-License-Identifier: Apache-2.0
+
+import asana
+import logging
+from typing import Dict, List
+
+from amundsen_application.base.base_issue_tracker_client import BaseIssueTrackerClient
+from amundsen_application.models.data_issue import DataIssue, Priority
+from amundsen_application.models.issue_results import IssueResults
+
+
+class AsanaClient(BaseIssueTrackerClient):
+
+    def __init__(self, issue_labels: List[str],
+                 issue_tracker_url: str,
+                 issue_tracker_user: str,
+                 issue_tracker_password: str,
+                 issue_tracker_project_id: int,
+                 issue_tracker_max_results: int) -> None:
+        self.issue_labels = issue_labels
+        self.asana_url = issue_tracker_url
+        self.asana_user = issue_tracker_user
+        self.asana_password = issue_tracker_password
+        self.asana_max_results = issue_tracker_max_results
+
+        self.asana_project_gid = issue_tracker_project_id
+        self.asana_client = asana.Client.access_token(issue_tracker_password)
+
+        asana_project = self.asana_client.projects.get_project(self.asana_project_gid)
+        self.asana_workspace_gid = asana_project['workspace']['gid']
+
+        self._setup_custom_fields()
+
+    def get_issues(self, table_uri: str) -> IssueResults:
+        """
+        :param table_uri: Table Uri ie databasetype://database/table
+        :return: Metadata of matching issues
+        """
+
+        table_parent_task_gid = self._get_parent_task_gid_for_table_uri(table_uri)
+
+        tasks = list(self.asana_client.tasks.get_subtasks_for_task(
+            table_parent_task_gid,
+            {
+                'opt_fields': [
+                    'name', 'completed', 'notes', 'custom_fields',
+                ]
+            }
+        ))
+
+        return IssueResults(
+            issues=[
+                self._asana_task_to_amundsen_data_issue(task) for task in tasks
+            ],
+            total=len(tasks),
+            all_issues_url=self._task_url(table_parent_task_gid),
+        )
+
+    def create_issue(self, table_uri: str, title: str, description: str) -> DataIssue:
+        """
+        Creates an issue in Jira
+        :param description: Description of the Jira issue
+        :param table_uri: Table Uri ie databasetype://database/table
+        :param title: Title of the Jira ticket
+        :return: Metadata about the newly created issue
+        """
+
+        table_parent_task_gid = self._get_parent_task_gid_for_table_uri(table_uri)
+
+        return self._asana_task_to_amundsen_data_issue(
+            self.asana_client.tasks.create_subtask_for_task(
+                table_parent_task_gid,
+                {
+                    'name': title,
+                    'notes': description,
+                }
+            )
+        )
+
+    def _setup_custom_fields(self) -> None:
+        TABLE_URI_FIELD_NAME = 'Table URI (Amundsen)'
+        PRIORITY_FIELD_NAME = 'Priority (Amundsen)'
+
+        custom_fields = \
+            self.asana_client.custom_field_settings.get_custom_field_settings_for_project(
+                self.asana_project_gid
+            )
+
+        custom_fields = {f['custom_field']['name']: f['custom_field'] for f in custom_fields}
+
+        if TABLE_URI_FIELD_NAME in custom_fields:
+            table_uri_field = custom_fields[TABLE_URI_FIELD_NAME]
+        else:
+            table_uri_field = self.asana_client.custom_fields.create_custom_field({
+                'workspace': self.asana_workspace_gid,
+                'name': TABLE_URI_FIELD_NAME,
+                'format': 'custom',
+                'resource_subtype': 'text',
+            })
+
+            self.asana_client.projects.add_custom_field_setting_for_project(
+                self.asana_project_gid,
+                {
+                    'custom_field': table_uri_field['gid'],
+                    'is_important': True,
+                }
+            )
+
+        if PRIORITY_FIELD_NAME in custom_fields:
+            priority_field = custom_fields[PRIORITY_FIELD_NAME]
+        else:
+            priority_field = self.asana_client.custom_fields.create_custom_field({
+                'workspace': self.asana_workspace_gid,
+                'name': PRIORITY_FIELD_NAME,
+                'format': 'custom',
+                'resource_subtype': 'enum',
+                'enum_options': [
+                    {
+                        'name': p.level
+                    } for p in Priority
+                ]
+            })
+
+            self.asana_client.projects.add_custom_field_setting_for_project(
+                self.asana_project_gid,
+                {
+                    'custom_field': priority_field['gid'],
+                    'is_important': True,
+                }
+            )
+
+        self.table_uri_field_gid = table_uri_field['gid']
+        self.priority_field_gid = priority_field['gid']
+
+    def _get_parent_task_gid_for_table_uri(self, table_uri: str) -> str:
+        table_parent_tasks = list(self.asana_client.tasks.search_tasks_for_workspace(
+            self.asana_workspace_gid,
+            {
+                'projects.any': [self.asana_project_gid],
+                'custom_fields.{}.value'.format(self.table_uri_field_gid): table_uri,
+            }
+        ))
+
+        # Create the parent task if it doesn't exist.
+        if len(table_parent_tasks) == 0:
+            table_parent_task = self.asana_client.tasks.create_task({
+                'name': table_uri,
+                'custom_fields': {
+                    self.table_uri_field_gid: table_uri,
+                },
+                'projects': [self.asana_project_gid],
+            })
+
+            return table_parent_task['gid']
+        else:
+            if len(table_parent_tasks) > 1:
+                logging.warn('There are currently two tasks with the name "{}"'.format(table_uri))
+
+            return table_parent_tasks[0]['gid']
+
+    def _task_url(self, task_gid: str) -> str:
+        return 'https://app.asana.com/0/{project_gid}/{task_gid}'.format(
+            project_gid=self.asana_project_gid, task_gid=task_gid
+        )
+
+    def _asana_task_to_amundsen_data_issue(self, task: Dict) -> DataIssue:
+        custom_fields = {f['gid']: f for f in task['custom_fields']}
+        priority_field = custom_fields[self.priority_field_gid]
+
+        priority = None
+        if priority_field.get('enum_value'):
+            priority = Priority.from_level(priority_field['enum_value']['name'])
+        else:
+            priority = Priority.P3
+
+        return DataIssue(
+            issue_key=task['gid'],
+            title=task['name'],
+            url=self._task_url(task['gid']),
+            status='closed' if task['completed'] else 'open',
+            priority=priority,
+        )

amundsen_application/proxy/issue_tracker_clients/jira_client.py

@@ -8,7 +8,7 @@ from flask import current_app as app
 
 from amundsen_application.base.base_issue_tracker_client import BaseIssueTrackerClient
 from amundsen_application.proxy.issue_tracker_clients.issue_exceptions import IssueConfigurationException
-from amundsen_application.models.data_issue import DataIssue
+from amundsen_application.models.data_issue import DataIssue, Priority
 from amundsen_application.models.issue_results import IssueResults
 
 import urllib.parse
@@ -132,7 +132,7 @@ class JiraClient(BaseIssueTrackerClient):
                          title=issue.fields.summary,
                          url=issue.permalink(),
                          status=issue.fields.status.name,
-                         priority=issue.fields.priority.name)
+                         priority=Priority.from_jira_severity(issue.fields.priority.name))
 
     def _generate_all_issues_url(self, table_uri: str, issues: List[DataIssue]) -> str:
         """

requirements.txt

@@ -79,5 +79,8 @@ flask-restful==0.3.8
 # SDK for JIRA
 jira==2.0.0
 
+# SDK for Asana
+asana==0.10.0
+
 # Retrying library
 retrying>=1.3.3,<2.0

tests/unit/api/issue/test_issue.py

@@ -6,7 +6,7 @@ import unittest
 from http import HTTPStatus
 from amundsen_application import create_app
 from amundsen_application.proxy.issue_tracker_clients.issue_exceptions import IssueConfigurationException
-from amundsen_application.models.data_issue import DataIssue
+from amundsen_application.models.data_issue import DataIssue, Priority
 from amundsen_application.models.issue_results import IssueResults
 
 local_app = create_app('amundsen_application.config.TestConfig', 'tests/templates')
@@ -31,7 +31,7 @@ class IssueTest(unittest.TestCase):
                                          title='title',
                                          url='http://somewhere',
                                              status='open',
-                                             priority='Major')
+                                             priority=Priority.P2)
         self.expected_issues = IssueResults(issues=[self.mock_data_issue],
                                             total=0,
                                             all_issues_url="http://moredata")

tests/unit/issue_tracker_clients/test_asana_client.py

+# Copyright Contributors to the Amundsen project.
+# SPDX-License-Identifier: Apache-2.0
+
+from unittest.mock import Mock
+
+import flask
+import unittest
+from amundsen_application.proxy.issue_tracker_clients.issue_exceptions import IssueConfigurationException
+from amundsen_application.proxy.issue_tracker_clients.asana_client import AsanaClient
+from amundsen_application.models.data_issue import DataIssue, Priority
+
+app = flask.Flask(__name__)
+app.config.from_object('amundsen_application.config.TestConfig')
+
+
+class AsanaClientTest(unittest.TestCase):
+
+    def setUp(self) -> None:
+        self.mock_issue_instance = DataIssue(issue_key='key',
+                                             title='some title',
+                                             url='http://somewhere',
+                                             status='open',
+                                             priority=Priority.P2)
+
+    @unittest.mock.patch('amundsen_application.proxy.issue_tracker_clients.asana_client.asana.Client')
+    def test_create_AsanaClient_validates_config(self, mock_asana_client: Mock) -> None:
+        with app.test_request_context():
+            try:
+                AsanaClient(
+                    issue_labels=[],
+                    issue_tracker_url='',
+                    issue_tracker_user='',
+                    issue_tracker_password='',
+                    issue_tracker_project_id=-1,
+                    issue_tracker_max_results=-1)
+
+            except IssueConfigurationException as e:
+                self.assertTrue(type(e), type(IssueConfigurationException))
+                self.assertTrue(e, 'The following config settings must be set for Asana: '
+                                   'ISSUE_TRACKER_URL, ISSUE_TRACKER_USER, ISSUE_TRACKER_PASSWORD, '
+                                   'ISSUE_TRACKER_PROJECT_ID')

tests/unit/issue_tracker_clients/test_jira_client.py

@@ -7,7 +7,7 @@ import flask
 import unittest
 from amundsen_application.proxy.issue_tracker_clients.issue_exceptions import IssueConfigurationException
 from amundsen_application.proxy.issue_tracker_clients.jira_client import JiraClient, SEARCH_STUB_ALL_ISSUES
-from amundsen_application.models.data_issue import DataIssue
+from amundsen_application.models.data_issue import DataIssue, Priority
 from jira import JIRAError
 from typing import Dict, List
 
@@ -41,7 +41,7 @@ class JiraClientTest(unittest.TestCase):
                                              title='some title',
                                              url='http://somewhere',
                                              status='open',
-                                             priority='Major')
+                                             priority=Priority.P2)
 
     @unittest.mock.patch('amundsen_application.proxy.issue_tracker_clients.jira_client.JIRA')
     def test_create_JiraClient_validates_config(self, mock_JIRA_client: Mock) -> None:
@@ -121,7 +121,7 @@ class JiraClientTest(unittest.TestCase):
                                      issue_tracker_password=app.config['ISSUE_TRACKER_PASSWORD'],
                                      issue_tracker_project_id=app.config['ISSUE_TRACKER_PROJECT_ID'],
                                      issue_tracker_max_results=app.config['ISSUE_TRACKER_MAX_RESULTS'])
-            issues = [DataIssue(issue_key='key', title='title', url='url', status='open', priority='Major')]
+            issues = [DataIssue(issue_key='key', title='title', url='url', status='open', priority=Priority.P2)]
             url = jira_client._generate_all_issues_url(table_uri="table", issues=issues)
             self.assertEqual(url, 'test_url/issues/?jql=test')
 

tests/unit/models/test_data_issue.py

@@ -2,7 +2,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import unittest
-from amundsen_application.models.data_issue import DataIssue
+from amundsen_application.models.data_issue import DataIssue, Priority
 
 
 class DataIssueTest(unittest.TestCase):
@@ -12,7 +12,7 @@ class DataIssueTest(unittest.TestCase):
         self.title = 'title'
         self.url = 'https://place'
         self.status = 'open'
-        self.priority = 'Major'
+        self.priority = Priority.P2
         self.maxDiff = None
 
     def test_mapping_priority(self) -> None:
@@ -22,13 +22,13 @@ class DataIssueTest(unittest.TestCase):
                                title=self.title,
                                url=self.url,
                                status=self.status,
-                               priority=self.priority)
-        self.assertEqual(data_issue.priority_display_name, expected_priority_display_name)
-        self.assertEqual(data_issue.priority_name, expected_priority_name)
-        self.assertEqual(data_issue.issue_key, self.issue_key)
-        self.assertEqual(data_issue.title, self.title)
-        self.assertEqual(data_issue.url, self.url)
-        self.assertEqual(data_issue.status, self.status)
+                               priority=self.priority).serialize()
+        self.assertEqual(data_issue['priority_display_name'], expected_priority_display_name)
+        self.assertEqual(data_issue['priority_name'], expected_priority_name)
+        self.assertEqual(data_issue['issue_key'], self.issue_key)
+        self.assertEqual(data_issue['title'], self.title)
+        self.assertEqual(data_issue['url'], self.url)
+        self.assertEqual(data_issue['status'], self.status)
 
     def test_mapping_priorty_missing(self) -> None:
         expected_priority_name = None  # type: ignore
@@ -37,10 +37,10 @@ class DataIssueTest(unittest.TestCase):
                                title=self.title,
                                url=self.url,
                                status=self.status,
-                               priority='missing')
-        self.assertEqual(data_issue.priority_display_name, expected_priority_display_name)
-        self.assertEqual(data_issue.priority_name, expected_priority_name)
-        self.assertEqual(data_issue.issue_key, self.issue_key)
-        self.assertEqual(data_issue.title, self.title)
-        self.assertEqual(data_issue.url, self.url)
-        self.assertEqual(data_issue.status, self.status)
+                               priority=None).serialize()
+        self.assertEqual(data_issue['priority_display_name'], expected_priority_display_name)
+        self.assertEqual(data_issue['priority_name'], expected_priority_name)
+        self.assertEqual(data_issue['issue_key'], self.issue_key)
+        self.assertEqual(data_issue['title'], self.title)
+        self.assertEqual(data_issue['url'], self.url)
+        self.assertEqual(data_issue['status'], self.status)