Add query and chart on Dashboard (#224)

* Add query and chart on Dashboard

* Update

* Increment version

* Update test case

databuilder/extractor/dashboard/mode_analytics/mode_dashboard_charts_extractor.py

+import logging
+
+from pyhocon import ConfigTree, ConfigFactory  # noqa: F401
+from typing import Any  # noqa: F401
+
+from databuilder import Scoped
+from databuilder.extractor.base_extractor import Extractor
+from databuilder.extractor.dashboard.mode_analytics.mode_dashboard_utils import ModeDashboardUtils
+from databuilder.rest_api.rest_api_query import RestApiQuery
+from databuilder.transformer.base_transformer import ChainedTransformer
+from databuilder.transformer.dict_to_model import DictToModel, MODEL_CLASS
+from databuilder.transformer.template_variable_substitution_transformer import \
+    TemplateVariableSubstitutionTransformer, FIELD_NAME, TEMPLATE
+
+LOGGER = logging.getLogger(__name__)
+
+
+class ModeDashboardChartsExtractor(Extractor):
+    """
+    A Extractor that extracts Dashboard charts
+
+    """
+
+    def init(self, conf):
+        # type: (ConfigTree) -> None
+        self._conf = conf
+
+        restapi_query = self._build_restapi_query()
+        self._extractor = ModeDashboardUtils.create_mode_rest_api_extractor(
+            restapi_query=restapi_query,
+            conf=self._conf
+        )
+
+        # Constructing URL using resource path via TemplateVariableSubstitutionTransformer
+        transformers = []
+        chart_url_transformer = TemplateVariableSubstitutionTransformer()
+        chart_url_transformer.init(
+            conf=Scoped.get_scoped_conf(self._conf, chart_url_transformer.get_scope()).with_fallback(
+                ConfigFactory.from_dict({FIELD_NAME: 'chart_url',
+                                         TEMPLATE: 'https://app.mode.com{chart_url}'})))
+
+        transformers.append(chart_url_transformer)
+
+        dict_to_model_transformer = DictToModel()
+        dict_to_model_transformer.init(
+            conf=Scoped.get_scoped_conf(self._conf, dict_to_model_transformer.get_scope()).with_fallback(
+                ConfigFactory.from_dict(
+                    {MODEL_CLASS: 'databuilder.models.dashboard.dashboard_chart.DashboardChart'})))
+        transformers.append(dict_to_model_transformer)
+
+        self._transformer = ChainedTransformer(transformers=transformers)
+
+    def extract(self):
+        # type: () -> Any
+
+        record = self._extractor.extract()
+        if not record:
+            return None
+
+        return self._transformer.transform(record=record)
+
+    def get_scope(self):
+        # type: () -> str
+        return 'extractor.mode_dashboard_chart'
+
+    def _build_restapi_query(self):
+        """
+        Build REST API Query. To get Mode Dashboard last execution, it needs to call three APIs (spaces API, reports
+        API, and run API) joining together.
+        :return: A RestApiQuery that provides Mode Dashboard execution (run)
+        """
+        # type: () -> RestApiQuery
+
+        spaces_query = ModeDashboardUtils.get_spaces_query_api(conf=self._conf)
+        params = ModeDashboardUtils.get_auth_params(conf=self._conf)
+
+        # Reports
+        # https://mode.com/developer/api-reference/analytics/reports/#listReportsInSpace
+        report_url_template = 'https://app.mode.com/api/{organization}/spaces/{dashboard_group_id}/reports'
+        json_path = '(_embedded.reports[*].token)'
+        field_names = ['dashboard_id']
+        reports_query = RestApiQuery(query_to_join=spaces_query, url=report_url_template, params=params,
+                                     json_path=json_path, field_names=field_names, skip_no_result=True)
+
+        queries_url_template = 'https://app.mode.com/api/{organization}/reports/{dashboard_id}/queries'
+        json_path = '_embedded.queries[*].[token,name]'
+        field_names = ['query_id', 'query_name']
+        query_names_query = RestApiQuery(query_to_join=reports_query, url=queries_url_template, params=params,
+                                         json_path=json_path, field_names=field_names, skip_no_result=True)
+
+        charts_url_template = 'https://app.mode.com/api/{organization}/reports/{dashboard_id}/queries/{query_id}/charts'
+        json_path = '(_embedded.charts[*].token) | (_embedded.charts[*]._links.report_viz_web.href)'
+        field_names = ['chart_id', 'chart_url']
+        chart_names_query = RestApiQuery(query_to_join=query_names_query, url=charts_url_template, params=params,
+                                         json_path=json_path, field_names=field_names, skip_no_result=True,
+                                         json_path_contains_or=True)
+
+        return chart_names_query

databuilder/extractor/dashboard/mode_analytics/mode_dashboard_queries_extractor.py

+import logging
+
+from pyhocon import ConfigTree, ConfigFactory  # noqa: F401
+from typing import Any  # noqa: F401
+
+from databuilder import Scoped
+from databuilder.extractor.base_extractor import Extractor
+from databuilder.extractor.dashboard.mode_analytics.mode_dashboard_utils import ModeDashboardUtils
+from databuilder.rest_api.rest_api_query import RestApiQuery
+from databuilder.transformer.base_transformer import ChainedTransformer
+from databuilder.transformer.dict_to_model import DictToModel, MODEL_CLASS
+from databuilder.transformer.template_variable_substitution_transformer import \
+    TemplateVariableSubstitutionTransformer, TEMPLATE, FIELD_NAME
+
+LOGGER = logging.getLogger(__name__)
+
+
+class ModeDashboardQueriesExtractor(Extractor):
+    """
+    A Extractor that extracts run (execution) status and timestamp.
+
+    """
+
+    def init(self, conf):
+        # type: (ConfigTree) -> None
+        self._conf = conf
+
+        restapi_query = self._build_restapi_query()
+        self._extractor = ModeDashboardUtils.create_mode_rest_api_extractor(
+            restapi_query=restapi_query,
+            conf=self._conf
+        )
+
+        # Constructing URL using several ID via TemplateVariableSubstitutionTransformer
+        transformers = []
+        variable_substitution_transformer = TemplateVariableSubstitutionTransformer()
+        variable_substitution_transformer.init(
+            conf=Scoped.get_scoped_conf(self._conf,
+                                        variable_substitution_transformer.get_scope()).with_fallback(
+                ConfigFactory.from_dict({FIELD_NAME: 'url',
+                                         TEMPLATE: 'https://app.mode.com/{organization}'
+                                                   '/reports/{dashboard_id}/queries/{query_id}'})))
+
+        transformers.append(variable_substitution_transformer)
+
+        dict_to_model_transformer = DictToModel()
+        dict_to_model_transformer.init(
+            conf=Scoped.get_scoped_conf(self._conf, dict_to_model_transformer.get_scope()).with_fallback(
+                ConfigFactory.from_dict(
+                    {MODEL_CLASS: 'databuilder.models.dashboard.dashboard_query.DashboardQuery'})))
+        transformers.append(dict_to_model_transformer)
+
+        self._transformer = ChainedTransformer(transformers=transformers)
+
+    def extract(self):
+        # type: () -> Any
+
+        record = self._extractor.extract()
+        if not record:
+            return None
+
+        return self._transformer.transform(record=record)
+
+    def get_scope(self):
+        # type: () -> str
+        return 'extractor.mode_dashboard_query'
+
+    def _build_restapi_query(self):
+        """
+        Build REST API Query. To get Mode Dashboard last execution, it needs to call three APIs (spaces API, reports
+        API, and queries API) joining together.
+        :return: A RestApiQuery that provides Mode Dashboard execution (run)
+        """
+        # type: () -> RestApiQuery
+
+        spaces_query = ModeDashboardUtils.get_spaces_query_api(conf=self._conf)
+        params = ModeDashboardUtils.get_auth_params(conf=self._conf)
+
+        # Reports
+        # https://mode.com/developer/api-reference/analytics/reports/#listReportsInSpace
+        url = 'https://app.mode.com/api/{organization}/spaces/{dashboard_group_id}/reports'
+        json_path = '(_embedded.reports[*].token)'
+        field_names = ['dashboard_id']
+        reports_query = RestApiQuery(query_to_join=spaces_query, url=url, params=params,
+                                     json_path=json_path, field_names=field_names, skip_no_result=True)
+
+        queries_url_template = 'https://app.mode.com/api/{organization}/reports/{dashboard_id}/queries'
+        json_path = '_embedded.queries[*].[token,name]'
+        field_names = ['query_id', 'query_name']
+        query_names_query = RestApiQuery(query_to_join=reports_query, url=queries_url_template, params=params,
+                                         json_path=json_path, field_names=field_names, skip_no_result=True)
+
+        return query_names_query

databuilder/models/dashboard/dashboard_chart.py

+import logging
+
+from typing import Optional, Dict, Any, Union, Iterator  # noqa: F401
+
+from databuilder.models.dashboard.dashboard_query import DashboardQuery
+from databuilder.models.neo4j_csv_serde import (
+    Neo4jCsvSerializable, NODE_LABEL, NODE_KEY, RELATION_START_KEY, RELATION_END_KEY, RELATION_START_LABEL,
+    RELATION_END_LABEL, RELATION_TYPE, RELATION_REVERSE_TYPE)
+
+LOGGER = logging.getLogger(__name__)
+
+
+class DashboardChart(Neo4jCsvSerializable):
+    """
+    A model that encapsulate Dashboard's charts
+    """
+    DASHBOARD_CHART_LABEL = 'Chart'
+    DASHBOARD_CHART_KEY_FORMAT = '{product}_dashboard://{cluster}.{dashboard_group_id}/' \
+                                 '{dashboard_id}/query/{query_id}/chart/{chart_id}'
+    CHART_RELATION_TYPE = 'HAS_CHART'
+    CHART_REVERSE_RELATION_TYPE = 'CHART_OF'
+
+    def __init__(self,
+                 dashboard_group_id,  # type: Optional[str]
+                 dashboard_id,  # type: Optional[str]
+                 query_id,  # type: str
+                 chart_id,  # type: str
+                 chart_name=None,  # type: Optional[str]
+                 chart_type=None,  # type: Optional[str]
+                 chart_url=None,  # type: Optional[str]
+                 product='',  # type: Optional[str]
+                 cluster='gold',  # type: str
+                 **kwargs
+                 ):
+        self._dashboard_group_id = dashboard_group_id
+        self._dashboard_id = dashboard_id
+        self._query_id = query_id
+        self._chart_id = chart_id if chart_id else chart_name
+        self._chart_name = chart_name
+        self._chart_type = chart_type
+        self._chart_url = chart_url
+        self._product = product
+        self._cluster = cluster
+        self._node_iterator = self._create_node_iterator()
+        self._relation_iterator = self._create_relation_iterator()
+
+    def create_next_node(self):
+        # type: () -> Union[Dict[str, Any], None]
+        try:
+            return next(self._node_iterator)
+        except StopIteration:
+            return None
+
+    def _create_node_iterator(self):  # noqa: C901
+        # type: () -> Iterator[[Dict[str, Any]]]
+        node = {
+            NODE_LABEL: DashboardChart.DASHBOARD_CHART_LABEL,
+            NODE_KEY: self._get_chart_node_key(),
+            'id': self._chart_id
+        }
+
+        if self._chart_name:
+            node['name'] = self._chart_name
+
+        if self._chart_type:
+            node['type'] = self._chart_type
+
+        if self._chart_url:
+            node['url'] = self._chart_url
+
+        yield node
+
+    def create_next_relation(self):
+        # type: () -> Union[Dict[str, Any], None]
+        try:
+            return next(self._relation_iterator)
+        except StopIteration:
+            return None
+
+    def _create_relation_iterator(self):
+        # type: () -> Iterator[[Dict[str, Any]]]
+        yield {
+            RELATION_START_LABEL: DashboardQuery.DASHBOARD_QUERY_LABEL,
+            RELATION_END_LABEL: DashboardChart.DASHBOARD_CHART_LABEL,
+            RELATION_START_KEY: DashboardQuery.DASHBOARD_QUERY_KEY_FORMAT.format(
+                product=self._product,
+                cluster=self._cluster,
+                dashboard_group_id=self._dashboard_group_id,
+                dashboard_id=self._dashboard_id,
+                query_id=self._query_id
+            ),
+            RELATION_END_KEY: self._get_chart_node_key(),
+            RELATION_TYPE: DashboardChart.CHART_RELATION_TYPE,
+            RELATION_REVERSE_TYPE: DashboardChart.CHART_REVERSE_RELATION_TYPE
+        }
+
+    def _get_chart_node_key(self):
+        return DashboardChart.DASHBOARD_CHART_KEY_FORMAT.format(
+            product=self._product,
+            cluster=self._cluster,
+            dashboard_group_id=self._dashboard_group_id,
+            dashboard_id=self._dashboard_id,
+            query_id=self._query_id,
+            chart_id=self._chart_id
+        )
+
+    def __repr__(self):
+        return 'DashboardChart({!r}, {!r}, {!r}, {!r}, {!r}, {!r}, {!r}, {!r}, {!r})'.format(
+            self._dashboard_group_id,
+            self._dashboard_id,
+            self._query_id,
+            self._chart_id,
+            self._chart_name,
+            self._chart_type,
+            self._chart_url,
+            self._product,
+            self._cluster
+        )

databuilder/models/dashboard/dashboard_query.py

+import logging
+
+from typing import Optional, Dict, Any, Union, Iterator  # noqa: F401
+
+from databuilder.models.dashboard.dashboard_metadata import DashboardMetadata
+from databuilder.models.neo4j_csv_serde import (
+    Neo4jCsvSerializable, NODE_LABEL, NODE_KEY, RELATION_START_KEY, RELATION_END_KEY, RELATION_START_LABEL,
+    RELATION_END_LABEL, RELATION_TYPE, RELATION_REVERSE_TYPE)
+
+LOGGER = logging.getLogger(__name__)
+
+
+class DashboardQuery(Neo4jCsvSerializable):
+    """
+    A model that encapsulate Dashboard's query name
+    """
+    DASHBOARD_QUERY_LABEL = 'Query'
+    DASHBOARD_QUERY_KEY_FORMAT = '{product}_dashboard://{cluster}.{dashboard_group_id}/' \
+                                 '{dashboard_id}/query/{query_id}'
+    DASHBOARD_QUERY_RELATION_TYPE = 'HAS_QUERY'
+    QUERY_DASHBOARD_RELATION_TYPE = 'QUERY_OF'
+
+    def __init__(self,
+                 dashboard_group_id,  # type: Optional[str]
+                 dashboard_id,  # type: Optional[str]
+                 query_name,  # type: str
+                 query_id=None,  # type: Optional[str]
+                 url='',  # type: Optional[str]
+                 product='',  # type: Optional[str]
+                 cluster='gold',  # type: str
+                 **kwargs
+                 ):
+        self._dashboard_group_id = dashboard_group_id
+        self._dashboard_id = dashboard_id
+        self._query_name = query_name
+        self._query_id = query_id if query_id else query_name
+        self._url = url
+        self._product = product
+        self._cluster = cluster
+        self._node_iterator = self._create_node_iterator()
+        self._relation_iterator = self._create_relation_iterator()
+
+    def create_next_node(self):
+        # type: () -> Union[Dict[str, Any], None]
+        try:
+            return next(self._node_iterator)
+        except StopIteration:
+            return None
+
+    def _create_node_iterator(self):  # noqa: C901
+        # type: () -> Iterator[[Dict[str, Any]]]
+        node = {
+            NODE_LABEL: DashboardQuery.DASHBOARD_QUERY_LABEL,
+            NODE_KEY: self._get_query_node_key(),
+            'id': self._query_id,
+            'name': self._query_name,
+        }
+
+        if self._url:
+            node['url'] = self._url
+
+        yield node
+
+    def create_next_relation(self):
+        # type: () -> Union[Dict[str, Any], None]
+        try:
+            return next(self._relation_iterator)
+        except StopIteration:
+            return None
+
+    def _create_relation_iterator(self):
+        # type: () -> Iterator[[Dict[str, Any]]]
+        yield {
+            RELATION_START_LABEL: DashboardMetadata.DASHBOARD_NODE_LABEL,
+            RELATION_END_LABEL: DashboardQuery.DASHBOARD_QUERY_LABEL,
+            RELATION_START_KEY: DashboardMetadata.DASHBOARD_KEY_FORMAT.format(
+                product=self._product,
+                cluster=self._cluster,
+                dashboard_group=self._dashboard_group_id,
+                dashboard_name=self._dashboard_id
+            ),
+            RELATION_END_KEY: self._get_query_node_key(),
+            RELATION_TYPE: DashboardQuery.DASHBOARD_QUERY_RELATION_TYPE,
+            RELATION_REVERSE_TYPE: DashboardQuery.QUERY_DASHBOARD_RELATION_TYPE
+        }
+
+    def _get_query_node_key(self):
+        return DashboardQuery.DASHBOARD_QUERY_KEY_FORMAT.format(
+            product=self._product,
+            cluster=self._cluster,
+            dashboard_group_id=self._dashboard_group_id,
+            dashboard_id=self._dashboard_id,
+            query_id=self._query_id
+        )
+
+    def __repr__(self):
+        return 'DashboardQuery({!r}, {!r}, {!r}, {!r}, {!r}, {!r}, {!r})'.format(
+            self._dashboard_group_id,
+            self._dashboard_id,
+            self._query_name,
+            self._query_id,
+            self._url,
+            self._product,
+            self._cluster
+        )

setup.py

@@ -2,8 +2,7 @@ import os
 from setuptools import setup, find_packages
 
 
-__version__ = '2.3.4'
-
+__version__ = '2.4.0'
 
 requirements_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'requirements.txt')
 with open(requirements_path) as requirements_file:

tests/unit/models/dashboard/test_dashboard_chart.py

+import unittest
+
+from databuilder.models.dashboard.dashboard_chart import DashboardChart
+from databuilder.models.neo4j_csv_serde import RELATION_START_KEY, RELATION_START_LABEL, RELATION_END_KEY, \
+    RELATION_END_LABEL, RELATION_TYPE, RELATION_REVERSE_TYPE
+
+
+class TestDashboardChart(unittest.TestCase):
+
+    def test_create_nodes(self):
+        # type: () -> None
+
+        dashboard_chart = DashboardChart(dashboard_group_id='dg_id',
+                                         dashboard_id='d_id',
+                                         query_id='q_id',
+                                         chart_id='c_id',
+                                         chart_name='c_name',
+                                         chart_type='bar',
+                                         chart_url='http://gold.foo/chart'
+                                         )
+
+        actual = dashboard_chart.create_next_node()
+        expected = {'name': 'c_name', 'type': 'bar', 'id': 'c_id', 'url': 'http://gold.foo/chart',
+                    'KEY': '_dashboard://gold.dg_id/d_id/query/q_id/chart/c_id', 'LABEL': 'Chart'}
+
+        self.assertDictEqual(expected, actual)
+        self.assertIsNone(dashboard_chart.create_next_node())
+
+        dashboard_chart = DashboardChart(dashboard_group_id='dg_id',
+                                         dashboard_id='d_id',
+                                         query_id='q_id',
+                                         chart_id='c_id',
+                                         chart_url='http://gold.foo.bar/'
+                                         )
+
+        actual2 = dashboard_chart.create_next_node()
+        expected2 = {'id': 'c_id', 'KEY': '_dashboard://gold.dg_id/d_id/query/q_id/chart/c_id', 'LABEL': 'Chart',
+                     'url': 'http://gold.foo.bar/'}
+        self.assertDictEqual(expected2, actual2)
+
+    def test_create_relation(self):
+        # type: () -> None
+        dashboard_chart = DashboardChart(dashboard_group_id='dg_id',
+                                         dashboard_id='d_id',
+                                         query_id='q_id',
+                                         chart_id='c_id',
+                                         chart_name='c_name',
+                                         chart_type='bar',
+                                         )
+
+        actual = dashboard_chart.create_next_relation()
+        expected = {RELATION_END_KEY: '_dashboard://gold.dg_id/d_id/query/q_id/chart/c_id',
+                    RELATION_START_LABEL: 'Query', RELATION_END_LABEL: 'Chart',
+                    RELATION_START_KEY: '_dashboard://gold.dg_id/d_id/query/q_id', RELATION_TYPE: 'HAS_CHART',
+                    RELATION_REVERSE_TYPE: 'CHART_OF'}
+
+        self.assertEqual(expected, actual)
+        self.assertIsNone(dashboard_chart.create_next_relation())

tests/unit/models/dashboard/test_dashboard_query.py

+import unittest
+
+from databuilder.models.dashboard.dashboard_query import DashboardQuery
+from databuilder.models.neo4j_csv_serde import NODE_KEY, \
+    NODE_LABEL, RELATION_START_KEY, RELATION_START_LABEL, RELATION_END_KEY, \
+    RELATION_END_LABEL, RELATION_TYPE, RELATION_REVERSE_TYPE
+
+
+class TestDashboardQuery(unittest.TestCase):
+
+    def test_create_nodes(self):
+        # type: () -> None
+
+        dashboard_query = DashboardQuery(dashboard_group_id='dg_id',
+                                         dashboard_id='d_id',
+                                         query_id='q_id',
+                                         query_name='q_name',
+                                         url='http://foo.bar/query/baz')
+
+        actual = dashboard_query.create_next_node()
+        expected = {'url': 'http://foo.bar/query/baz', 'name': 'q_name', 'id': 'q_id',
+                    NODE_KEY: '_dashboard://gold.dg_id/d_id/query/q_id',
+                    NODE_LABEL: DashboardQuery.DASHBOARD_QUERY_LABEL}
+
+        self.assertEqual(expected, actual)
+
+    def test_create_relation(self):
+        # type: () -> None
+        dashboard_query = DashboardQuery(dashboard_group_id='dg_id',
+                                         dashboard_id='d_id',
+                                         query_id='q_id',
+                                         query_name='q_name')
+
+        actual = dashboard_query.create_next_relation()
+        expected = {RELATION_END_KEY: '_dashboard://gold.dg_id/d_id/query/q_id', RELATION_START_LABEL: 'Dashboard',
+                    RELATION_END_LABEL: DashboardQuery.DASHBOARD_QUERY_LABEL,
+                    RELATION_START_KEY: '_dashboard://gold.dg_id/d_id', RELATION_TYPE: 'HAS_QUERY',
+                    RELATION_REVERSE_TYPE: 'QUERY_OF'}
+
+        self.assertEqual(expected, actual)