feat: Add Query text to Dashboard Query model (#273)

* Add Query text to Dashboard Query model

* Update

* Update

* Update

databuilder/extractor/dashboard/mode_analytics/mode_dashboard_queries_extractor.py

@@ -10,6 +10,8 @@ from databuilder.rest_api.mode_analytics.mode_paginated_rest_api_query import Mo
 from databuilder.rest_api.rest_api_query import RestApiQuery
 from databuilder.transformer.base_transformer import ChainedTransformer
 from databuilder.transformer.dict_to_model import DictToModel, MODEL_CLASS
+from databuilder.transformer.regex_str_replace_transformer import RegexStrReplaceTransformer, \
+    REGEX_REPLACE_TUPLE_LIST, ATTRIBUTE_NAME
 from databuilder.transformer.template_variable_substitution_transformer import \
     TemplateVariableSubstitutionTransformer, TEMPLATE, FIELD_NAME
 
@@ -44,6 +46,14 @@ class ModeDashboardQueriesExtractor(Extractor):
 
         transformers.append(variable_substitution_transformer)
 
+        # Escape backslash as it breaks Cypher statement.
+        replace_transformer = RegexStrReplaceTransformer()
+        replace_transformer.init(
+            conf=Scoped.get_scoped_conf(self._conf, replace_transformer.get_scope()).with_fallback(
+                ConfigFactory.from_dict(
+                    {REGEX_REPLACE_TUPLE_LIST: [('\\', '\\\\')], ATTRIBUTE_NAME: 'query_text'})))
+        transformers.append(replace_transformer)
+
         dict_to_model_transformer = DictToModel()
         dict_to_model_transformer.init(
             conf=Scoped.get_scoped_conf(self._conf, dict_to_model_transformer.get_scope()).with_fallback(
@@ -86,8 +96,8 @@ class ModeDashboardQueriesExtractor(Extractor):
                                                   json_path=json_path, field_names=field_names, skip_no_result=True)
 
         queries_url_template = 'https://app.mode.com/api/{organization}/reports/{dashboard_id}/queries'
-        json_path = '_embedded.queries[*].[token,name]'
-        field_names = ['query_id', 'query_name']
+        json_path = '_embedded.queries[*].[token,name,raw_query]'
+        field_names = ['query_id', 'query_name', 'query_text']
         query_names_query = RestApiQuery(query_to_join=reports_query, url=queries_url_template, params=params,
                                          json_path=json_path, field_names=field_names, skip_no_result=True)
 

databuilder/models/dashboard/dashboard_query.py

@@ -26,6 +26,7 @@ class DashboardQuery(Neo4jCsvSerializable):
                  query_name,  # type: str
                  query_id=None,  # type: Optional[str]
                  url='',  # type: Optional[str]
+                 query_text=None,  # type: Optional[str]
                  product='',  # type: Optional[str]
                  cluster='gold',  # type: str
                  **kwargs
@@ -35,6 +36,7 @@ class DashboardQuery(Neo4jCsvSerializable):
         self._query_name = query_name
         self._query_id = query_id if query_id else query_name
         self._url = url
+        self._query_text = query_text
         self._product = product
         self._cluster = cluster
         self._node_iterator = self._create_node_iterator()
@@ -59,6 +61,9 @@ class DashboardQuery(Neo4jCsvSerializable):
         if self._url:
             node['url'] = self._url
 
+        if self._query_text:
+            node['query_text'] = self._query_text
+
         yield node
 
     def create_next_relation(self):
@@ -94,12 +99,13 @@ class DashboardQuery(Neo4jCsvSerializable):
         )
 
     def __repr__(self):
-        return 'DashboardQuery({!r}, {!r}, {!r}, {!r}, {!r}, {!r}, {!r})'.format(
+        return 'DashboardQuery({!r}, {!r}, {!r}, {!r}, {!r}, {!r}, {!r}, {!r})'.format(
             self._dashboard_group_id,
             self._dashboard_id,
             self._query_name,
             self._query_id,
             self._url,
+            self._query_text,
             self._product,
             self._cluster
         )

databuilder/transformer/regex_str_replace_transformer.py

@@ -28,6 +28,10 @@ class RegexStrReplaceTransformer(Transformer):
 
     def transform(self, record):
         # type: (Any) -> Any
+
+        if isinstance(record, dict):
+            val = record.get(self._attribute_name)
+        else:
             val = getattr(record, self._attribute_name)
 
         if val is None or not isinstance(val, six.string_types):
@@ -40,7 +44,11 @@ class RegexStrReplaceTransformer(Transformer):
         for regex_replace_tuple in self._regex_replace_tuples:
             val = val.replace(regex_replace_tuple[0], regex_replace_tuple[1])
 
+        if isinstance(record, dict):
+            record[self._attribute_name] = val
+        else:
             setattr(record, self._attribute_name, val)
+
         return record
 
     def get_scope(self):

setup.py

@@ -2,7 +2,7 @@ import os
 from setuptools import setup, find_packages
 
 
-__version__ = '2.5.17'
+__version__ = '2.5.18'
 
 requirements_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'requirements.txt')
 with open(requirements_path) as requirements_file:

tests/unit/models/dashboard/test_dashboard_query.py

@@ -15,10 +15,12 @@ class TestDashboardQuery(unittest.TestCase):
                                          dashboard_id='d_id',
                                          query_id='q_id',
                                          query_name='q_name',
-                                         url='http://foo.bar/query/baz')
+                                         url='http://foo.bar/query/baz',
+                                         query_text='SELECT * FROM foo.bar')
 
         actual = dashboard_query.create_next_node()
         expected = {'url': 'http://foo.bar/query/baz', 'name': 'q_name', 'id': 'q_id',
+                    'query_text': 'SELECT * FROM foo.bar',
                     NODE_KEY: '_dashboard://gold.dg_id/d_id/query/q_id',
                     NODE_LABEL: DashboardQuery.DASHBOARD_QUERY_LABEL}
 

tests/unit/transformer/test_regex_str_replace_transformer.py

@@ -2,7 +2,8 @@ import unittest
 
 from pyhocon import ConfigFactory
 
-from databuilder.transformer.regex_str_replace_transformer import RegexStrReplaceTransformer
+from databuilder.transformer.regex_str_replace_transformer import RegexStrReplaceTransformer, \
+    REGEX_REPLACE_TUPLE_LIST, ATTRIBUTE_NAME
 
 
 class TestRegexReplacement(unittest.TestCase):
@@ -37,8 +38,8 @@ class TestRegexReplacement(unittest.TestCase):
     def _default_test_transformer(self):
         # type: () -> RegexStrReplaceTransformer
         config = ConfigFactory.from_dict({
-            'regex_replace_tuple_list': [('a', 'b'), ('c', 'a')],
-            'attribute_name': 'val'
+            REGEX_REPLACE_TUPLE_LIST: [('a', 'b'), ('c', 'a')],
+            ATTRIBUTE_NAME: 'val'
         })
 
         transformer = RegexStrReplaceTransformer()
@@ -46,6 +47,22 @@ class TestRegexReplacement(unittest.TestCase):
 
         return transformer
 
+    def test_dict_replace(self):
+        # type: () -> None
+        config = ConfigFactory.from_dict({
+            REGEX_REPLACE_TUPLE_LIST: [('\\', '\\\\')],
+            ATTRIBUTE_NAME: 'val'
+        })
+
+        transformer = RegexStrReplaceTransformer()
+        transformer.init(config)
+
+        d = {'val': '\\'}
+
+        actual = transformer.transform(d)
+
+        self.assertEqual({'val': '\\\\'}, actual)
+
 
 class Foo(object):
     def __init__(self, val):