Dashboard image preview (#423)

* Initial checkin of Dashboard image preview

* Update

* Update

* Update

* Update

* flake8 and mypy

* flake8 and mypy

* Update

.gitignore

@@ -28,6 +28,8 @@ node_modules/
 jest
 .coverage
 coverage
+**/coverage.xml
+**/htmlcov/**
 
 # IntelliJ
 *.iml

amundsen_application/__init__.py

@@ -15,6 +15,7 @@ from amundsen_application.api.mail.v0 import mail_blueprint
 from amundsen_application.api.metadata.v0 import metadata_blueprint
 from amundsen_application.api.preview.v0 import preview_blueprint
 from amundsen_application.api.search.v0 import search_blueprint
+from amundsen_application.api.preview.dashboard.v0 import dashboard_preview_blueprint
 from amundsen_application.api.issue.issue import IssueAPI, IssuesAPI
 
 
@@ -71,6 +72,7 @@ def create_app(config_module_class: str, template_folder: str = None) -> Flask:
     app.register_blueprint(preview_blueprint)
     app.register_blueprint(search_blueprint)
     app.register_blueprint(api_bp)
+    app.register_blueprint(dashboard_preview_blueprint)
     init_routes(app)
 
     init_custom_routes = app.config.get('INIT_CUSTOM_ROUTES')

amundsen_application/api/preview/dashboard/v0.py

+import io
+import logging
+from http import HTTPStatus
+
+from flask import send_file, jsonify, make_response, Response, current_app as app
+from flask.blueprints import Blueprint
+
+from amundsen_application.dashboard_preview.preview_factory_method import DefaultPreviewMethodFactory, \
+    BasePreviewMethodFactory
+
+LOGGER = logging.getLogger(__name__)
+PREVIEW_FACTORY: BasePreviewMethodFactory = None  # type: ignore
+
+dashboard_preview_blueprint = Blueprint('dashboard_preview', __name__, url_prefix='/api/dashboard_preview/v0')
+
+
+def initialize_preview_factory_class() -> None:
+    """
+    Instantiates Preview factory class and assign it to PREVIEW_FACTORY
+    :return: None
+    """
+    global PREVIEW_FACTORY
+
+    PREVIEW_FACTORY = app.config['DASHBOARD_PREVIEW_FACTORY']
+    if not PREVIEW_FACTORY:
+        PREVIEW_FACTORY = DefaultPreviewMethodFactory()
+
+    LOGGER.info('Using {} for Dashboard'.format(PREVIEW_FACTORY))
+
+
+@dashboard_preview_blueprint.route('/dashboard/<path:uri>/preview.jpg', methods=['GET'])
+def get_preview_image(uri: str) -> Response:
+    """
+    Provides preview image of Dashboard which can be cached for a day (by default).
+    :return:
+    """
+
+    if not PREVIEW_FACTORY:
+        LOGGER.info('Initializing Dashboard PREVIEW_FACTORY')
+        initialize_preview_factory_class()
+
+    preview_client = PREVIEW_FACTORY.get_instance(uri=uri)
+    try:
+        return send_file(io.BytesIO(preview_client.get_preview_image(uri=uri)),
+                         mimetype='image/jpeg',
+                         cache_timeout=app.config['DASHBOARD_PREVIEW_IMAGE_CACHE_MAX_AGE_SECONDS'])
+    except FileNotFoundError as fne:
+        return make_response(jsonify({'msg': fne.args[0]}), HTTPStatus.NOT_FOUND)
+    except Exception as e:
+        LOGGER.exception('Unexpected failure on get_preview_image')
+        return make_response(jsonify({'msg': 'Encountered exception: ' + str(e)}), HTTPStatus.INTERNAL_SERVER_ERROR)

amundsen_application/base/base_preview.py

+from abc import ABCMeta, abstractmethod
+
+
+class BasePreview(metaclass=ABCMeta):
+    """
+    A Preview interface for other product to implement. For example, see ModePreview.
+    """
+
+    @abstractmethod
+    def get_preview_image(self, *, uri: str) -> bytes:
+        """
+        Returns image bytes given URI
+        :param uri:
+        :return:
+        :raises: FileNotFound when either Report is not available or Preview image is not available
+        """
+        pass

amundsen_application/config.py

@@ -71,6 +71,16 @@ class Config:
     AUTH_USER_METHOD: Optional[Callable[[Flask], User]] = None
     GET_PROFILE_URL = None
 
+    # For additional preview client, register more at DefaultPreviewMethodFactory.__init__()
+    # For any private preview client, use custom factory that implements BasePreviewMethodFactory
+    DASHBOARD_PREVIEW_FACTORY = None  # By default DefaultPreviewMethodFactory will be used.
+    DASHBOARD_PREVIEW_IMAGE_CACHE_MAX_AGE_SECONDS = 60 * 60 * 24 * 1  # 1 day
+
+    CREDENTIALS_MODE_ADMIN_TOKEN = os.getenv('CREDENTIALS_MODE_ADMIN_TOKEN', None)
+    CREDENTIALS_MODE_ADMIN_PASSWORD = os.getenv('CREDENTIALS_MODE_ADMIN_PASSWORD', None)
+    MODE_ORGANIZATION = None
+    MODE_REPORT_URL_TEMPLATE = None
+
 
 class LocalConfig(Config):
     DEBUG = False

amundsen_application/dashboard_preview/mode_preview.py

+import logging
+from typing import Optional, Any
+
+import requests
+from flask import has_app_context, current_app as app
+from requests.auth import HTTPBasicAuth
+from retrying import retry
+
+from amundsen_application.base.base_preview import BasePreview
+
+LOGGER = logging.getLogger(__name__)
+DEFAULT_REPORT_URL_TEMPLATE = 'https://app.mode.com/api/{organization}/reports/{dashboard_id}'
+
+
+def _validate_not_none(var: Any, var_name: str) -> Any:
+    if not var:
+        raise ValueError('{} is missing'.format(var_name))
+    return var
+
+
+class ModePreview(BasePreview):
+    """
+    A class to get Mode Dashboard preview image
+    """
+
+    def __init__(self, *,
+                 access_token: Optional[str] = None,
+                 password: Optional[str] = None,
+                 organization: Optional[str] = None,
+                 report_url_template: Optional[str] = None):
+        self._access_token = access_token if access_token else app.config['CREDENTIALS_MODE_ADMIN_TOKEN']
+        _validate_not_none(self._access_token, 'access_token')
+        self._password = password if password else app.config['CREDENTIALS_MODE_ADMIN_PASSWORD']
+        _validate_not_none(self._password, 'password')
+        self._organization = organization if organization else app.config['MODE_ORGANIZATION']
+        _validate_not_none(self._organization, 'organization')
+
+        self._report_url_template = report_url_template if report_url_template else DEFAULT_REPORT_URL_TEMPLATE
+
+        if has_app_context() and app.config['MODE_REPORT_URL_TEMPLATE'] is not None:
+            self._report_url_template = app.config['MODE_REPORT_URL_TEMPLATE']
+
+    @retry(stop_max_attempt_number=3, wait_random_min=500, wait_random_max=1000)
+    def get_preview_image(self, *, uri: str) -> bytes:
+        """
+        Retrieves short lived URL that provides Mode report preview, downloads it and returns it's bytes
+        :param uri:
+        :return: image bytes
+        """
+        url = self._get_preview_image_url(uri=uri)
+        r = requests.get(url, allow_redirects=True)
+        r.raise_for_status()
+
+        return r.content
+
+    def _get_preview_image_url(self, *, uri: str) -> str:
+        url = self._report_url_template.format(organization=self._organization, dashboard_id=uri.split('/')[-1])
+
+        LOGGER.info('Calling URL {} to fetch preview image URL'.format(url))
+        response = requests.get(url, auth=HTTPBasicAuth(self._access_token, self._password))
+        if response.status_code == 404:
+            raise FileNotFoundError('Dashboard {} not found. Possibly has been deleted.'.format(uri))
+
+        response.raise_for_status()
+
+        web_preview_image_key = 'web_preview_image'
+        result = response.json()
+
+        if web_preview_image_key not in result:
+            raise FileNotFoundError('No preview image available on {}'.format(uri))
+
+        return result[web_preview_image_key]

amundsen_application/dashboard_preview/preview_factory_method.py

+import logging
+from abc import ABCMeta, abstractmethod
+
+from amundsen_application.base.base_preview import BasePreview
+from amundsen_application.dashboard_preview.mode_preview import ModePreview
+
+LOGGER = logging.getLogger(__name__)
+
+
+class BasePreviewMethodFactory(metaclass=ABCMeta):
+
+    @abstractmethod
+    def get_instance(self, *, uri: str) -> BasePreview:
+        """
+        Provides an instance of BasePreview based on uri
+        :param uri:
+        :return:
+        """
+        pass
+
+
+class DefaultPreviewMethodFactory(BasePreviewMethodFactory):
+
+    def __init__(self) -> None:
+        # Register preview clients here. Key: product, Value: BasePreview implementation
+        self._object_map = {
+            'mode': ModePreview()
+        }
+        LOGGER.info('Supported products: {}'.format(list(self._object_map.keys())))
+
+    def get_instance(self, *, uri: str) -> BasePreview:
+        product = self.get_product(uri=uri)
+
+        if product in self._object_map:
+            return self._object_map[product]
+
+        raise NotImplementedError('Product {} is not supported'.format(product))
+
+    def get_product(self, *, uri: str) -> str:
+        return uri.split('_')[0]

requirements.txt

@@ -78,3 +78,6 @@ flask-restful==0.3.7
 
 # SDK for JIRA
 jira==2.0.0
+
+# Retrying library
+retrying>=1.3.3,<2.0

setup.py

@@ -34,7 +34,7 @@ requirements_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'r
 with open(requirements_path) as requirements_file:
     requirements = requirements_file.readlines()
 
-__version__ = '2.1.1'
+__version__ = '2.2.0'
 
 
 setup(

tests/unit/api/preview/dashboard/test_v0.py

+import unittest
+
+from flask import current_app
+from http import HTTPStatus
+
+
+from amundsen_application import create_app
+from amundsen_application.api.preview.dashboard import v0
+from amundsen_application.base.base_preview import BasePreview
+from amundsen_application.dashboard_preview.preview_factory_method import DefaultPreviewMethodFactory,\
+    BasePreviewMethodFactory
+
+from unittest.mock import MagicMock
+
+
+class TestV0(unittest.TestCase):
+
+    def setUp(self) -> None:
+        self.app = create_app(config_module_class='amundsen_application.config.LocalConfig')
+        self.app_context = self.app.app_context()
+        self.app_context.push()
+        self.app.config['CREDENTIALS_MODE_ADMIN_TOKEN'] = 'CREDENTIALS_MODE_ADMIN_TOKEN'
+        self.app.config['CREDENTIALS_MODE_ADMIN_PASSWORD'] = 'CREDENTIALS_MODE_ADMIN_PASSWORD'
+        self.app.config['MODE_ORGANIZATION'] = 'foo'
+
+    def tearDown(self) -> None:
+        self.app_context.pop()
+
+    def test_app_exists(self) -> None:
+        self.assertFalse(current_app is None)
+
+    def test_v0_default_initialize_preview_factory_class(self) -> None:
+        v0.initialize_preview_factory_class()
+        self.assertTrue(isinstance(v0.PREVIEW_FACTORY, DefaultPreviewMethodFactory))
+
+    def test_v0_initialize_preview_factory_class(self) -> None:
+        self.app.config['DASHBOARD_PREVIEW_FACTORY'] = DummyPreviewMethodFactory()
+
+        v0.initialize_preview_factory_class()
+        self.assertTrue(isinstance(v0.PREVIEW_FACTORY, DummyPreviewMethodFactory))
+
+    def test_get_preview_image_not_available(self) -> None:
+        mock_factory = MagicMock()
+        mock_factory.get_instance.return_value.get_preview_image.side_effect = FileNotFoundError('foo')
+
+        self.app.config['DASHBOARD_PREVIEW_FACTORY'] = mock_factory
+        v0.initialize_preview_factory_class()
+
+        response = v0.get_preview_image(uri='foo')
+        self.assertEqual(response.status_code, HTTPStatus.NOT_FOUND)
+
+    def test_get_preview_image_failure(self) -> None:
+        mock_factory = MagicMock()
+        mock_factory.get_instance.return_value.get_preview_image.side_effect = Exception()
+
+        self.app.config['DASHBOARD_PREVIEW_FACTORY'] = mock_factory
+        v0.initialize_preview_factory_class()
+
+        response = v0.get_preview_image(uri='foo')
+        self.assertEqual(response.status_code, HTTPStatus.INTERNAL_SERVER_ERROR)
+
+
+class DummyPreviewMethodFactory(BasePreviewMethodFactory):
+    def get_instance(self, *, uri: str) -> BasePreview:
+        pass

tests/unit/dashboard_preview/test_mode_preview.py

+# flake8: noqa
+import unittest
+from unittest.mock import patch
+
+import requests
+from requests.auth import HTTPBasicAuth
+
+from amundsen_application.dashboard_preview.mode_preview import ModePreview, DEFAULT_REPORT_URL_TEMPLATE
+
+ACCESS_TOKEN = 'token'
+PASSWORD = 'password'
+ORGANIZATION = 'foo'
+
+
+class TestModePreview(unittest.TestCase):
+
+    def test_get_preview_image(self) -> None:
+        with patch.object(ModePreview, '_get_preview_image_url') as mock_get_preview_image_url,\
+             patch.object(requests, 'get') as mock_get:
+            mock_get_preview_image_url.return_value = 'http://foo.bar/image.jpeg'
+            mock_get.return_value.content = b'bar'
+
+            preview = ModePreview(access_token='token', password='password', organization='foo')
+
+            mode_dashboard_uri = 'mode_dashboard://gold.dg/d_id'
+            actual = preview.get_preview_image(uri=mode_dashboard_uri)
+
+            self.assertEqual(b'bar', actual)
+            mock_get.assert_called_with('http://foo.bar/image.jpeg', allow_redirects=True)
+
+    def test_get_preview_image_url(self) -> None:
+        with patch.object(requests, 'get') as mock_get:
+            mock_get.return_value.json.return_value = {'web_preview_image': 'http://foo.bar/image.jpeg'}
+
+            preview = ModePreview(access_token='token', password='password', organization='foo')
+            mode_dashboard_uri = 'mode_dashboard://gold.dg/d_id'
+            url = preview._get_preview_image_url(uri=mode_dashboard_uri)
+
+            self.assertEqual(url, 'http://foo.bar/image.jpeg')
+
+            expected_url = DEFAULT_REPORT_URL_TEMPLATE.format(organization=ORGANIZATION, dashboard_id='d_id')
+            mock_get.assert_called_with(expected_url, auth=HTTPBasicAuth(ACCESS_TOKEN, PASSWORD))

tests/unit/dashboard_preview/test_preview_factory_method.py

+# flake8: noqa
+import unittest
+
+from flask import current_app
+
+from amundsen_application import create_app
+from amundsen_application.dashboard_preview.mode_preview import ModePreview
+from amundsen_application.dashboard_preview.preview_factory_method import DefaultPreviewMethodFactory
+
+
+class TestDefaultPreviewFactory(unittest.TestCase):
+
+    def setUp(self) -> None:
+        self.app = create_app(config_module_class='amundsen_application.config.LocalConfig')
+        self.app_context = self.app.app_context()
+        self.app_context.push()
+        self.app.config['CREDENTIALS_MODE_ADMIN_TOKEN'] = 'CREDENTIALS_MODE_ADMIN_TOKEN'
+        self.app.config['CREDENTIALS_MODE_ADMIN_PASSWORD'] = 'CREDENTIALS_MODE_ADMIN_PASSWORD'
+        self.app.config['MODE_ORGANIZATION'] = 'foo'
+
+    def tearDown(self) -> None:
+        self.app_context.pop()
+
+    def test_app_exists(self) -> None:
+        self.assertFalse(current_app is None)
+
+    def test_factory(self) -> None:
+        factory = DefaultPreviewMethodFactory()
+        actual = factory.get_instance(uri='mode_dashboard://gold.dg_id/d_id')
+
+        self.assertTrue(isinstance(actual, ModePreview))
+
+        try:
+            factory.get_instance(uri='tableau_dashboard://foo.bar/baz')
+            self.assertTrue(False, 'Should have failed for we currently do not support Tableau')
+        except Exception:
+            pass