Adding Mode dashboard usage extractor and generic loader (#225)

* Adding Mode dashboard usage extractor and generic loader

* Update

* Increment version

databuilder/extractor/dashboard/mode_analytics/mode_dashboard_extractor.py

@@ -13,10 +13,6 @@ from databuilder.transformer.timestamp_string_to_epoch import TimestampStringToE
 from databuilder.transformer.template_variable_substitution_transformer import \
     TemplateVariableSubstitutionTransformer, TEMPLATE, FIELD_NAME as VAR_FIELD_NAME
 
-# CONFIG KEYS
-ORGANIZATION = 'organization'
-MODE_ACCESS_TOKEN = 'mode_user_token'
-MODE_PASSWORD_TOKEN = 'mode_password_token'
 
 LOGGER = logging.getLogger(__name__)
 

databuilder/extractor/dashboard/mode_analytics/mode_dashboard_usage_extractor.py

+import logging
+
+from pyhocon import ConfigTree  # noqa: F401
+from typing import Any  # noqa: F401
+
+from databuilder.extractor.base_extractor import Extractor
+from databuilder.extractor.dashboard.mode_analytics.mode_dashboard_utils import ModeDashboardUtils
+from databuilder.rest_api.rest_api_query import RestApiQuery
+
+
+LOGGER = logging.getLogger(__name__)
+
+
+class ModeDashboardUsageExtractor(Extractor):
+    """
+    A Extractor that extracts Mode dashboard's accumulated view count
+    """
+
+    def init(self, conf):
+        # type: (ConfigTree) -> None
+
+        self._conf = conf
+
+        restapi_query = self._build_restapi_query()
+        self._extractor = ModeDashboardUtils.create_mode_rest_api_extractor(restapi_query=restapi_query,
+                                                                            conf=self._conf)
+
+    def extract(self):
+        # type: () -> Any
+
+        return self._extractor.extract()
+
+    def get_scope(self):
+        # type: () -> str
+
+        return 'extractor.mode_dashboard_usage'
+
+    def _build_restapi_query(self):
+        """
+        Build REST API Query. To get Mode Dashboard usage, it needs to call two APIs (spaces API and reports
+        API) joining together.
+        :return: A RestApiQuery that provides Mode Dashboard metadata
+        """
+        # type: () -> RestApiQuery
+
+        # https://mode.com/developer/api-reference/analytics/reports/#listReportsInSpace
+        reports_url_template = 'https://app.mode.com/api/{organization}/spaces/{dashboard_group_id}/reports'
+
+        spaces_query = ModeDashboardUtils.get_spaces_query_api(conf=self._conf)
+        params = ModeDashboardUtils.get_auth_params(conf=self._conf)
+
+        # Reports
+        # JSONPATH expression. it goes into array which is located in _embedded.reports and then extracts token,
+        # and view_count
+        json_path = '_embedded.reports[*].[token,view_count]'
+        field_names = ['dashboard_id', 'accumulated_view_count']
+        reports_query = RestApiQuery(query_to_join=spaces_query, url=reports_url_template, params=params,
+                                     json_path=json_path, field_names=field_names, skip_no_result=True)
+        return reports_query

databuilder/loader/generic_loader.py

+import logging
+
+from pyhocon import ConfigTree  # noqa: F401
+from typing import Optional, Any  # noqa: F401
+
+from databuilder.loader.base_loader import Loader
+
+LOGGER = logging.getLogger(__name__)
+
+CALLBACK_FUNCTION = 'callback_function'
+
+
+def log_call_back(record):
+    """
+    A Sample callback function. Implement any function follows this function's signature to fit your needs.
+    :param record:
+    :return:
+    """
+    LOGGER.info('record: {}'.format(record))
+
+
+class GenericLoader(Loader):
+    """
+    Loader class to call back a function provided by user
+    """
+
+    def init(self, conf):
+        # type: (ConfigTree) -> None
+        """
+        Initialize file handlers from conf
+        :param conf:
+        """
+        self.conf = conf
+        self._callback_func = self.conf.get(CALLBACK_FUNCTION, log_call_back)
+
+    def load(self, record):
+        # type: (Optional[Any]) -> None
+        """
+        Write record to function
+        :param record:
+        :return:
+        """
+        if not record:
+            return
+
+        self._callback_func(record)
+
+    def close(self):
+        # type: () -> None
+        pass
+
+    def get_scope(self):
+        # type: () -> str
+        return "loader.generic"

setup.py

@@ -2,7 +2,7 @@ import os
 from setuptools import setup, find_packages
 
 
-__version__ = '2.3.3'
+__version__ = '2.3.4'
 
 
 requirements_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'requirements.txt')

tests/unit/loader/test_generic_loader.py

+import unittest
+
+from mock import MagicMock
+from pyhocon import ConfigFactory
+
+from databuilder.loader.generic_loader import GenericLoader, CALLBACK_FUNCTION
+
+
+class TestGenericLoader(unittest.TestCase):
+
+    def test_loading(self):
+        # type: () -> None
+
+        loader = GenericLoader()
+        callback_func = MagicMock()
+        loader.init(conf=ConfigFactory.from_dict({
+            CALLBACK_FUNCTION: callback_func
+        }))
+
+        loader.load({'foo': 'bar'})
+        loader.close()
+
+        callback_func.assert_called_once()
+
+    def test_none_loading(self):
+        # type: () -> None
+
+        loader = GenericLoader()
+        callback_func = MagicMock()
+        loader.init(conf=ConfigFactory.from_dict({
+            CALLBACK_FUNCTION: callback_func
+        }))
+
+        loader.load(None)
+        loader.close()
+
+        callback_func.assert_not_called()