Added created timestamp and last modified timestamp on Dashboard (#207)

* Added created timestamp and last modified timestamp on Dashboard

* Remove owner and reload time from dashboard_metadata

* Move dashboard_metadata under dashboard package

* Update

* Update

* Added dashboard group url and dashboard url

* Added cluster node in Dashboard

databuilder/extractor/dashboard/mode_analytics/mode_dashboard_extractor.py

@@ -3,10 +3,15 @@ import logging
 from pyhocon import ConfigTree, ConfigFactory  # noqa: F401
 from typing import Any  # noqa: F401
 
+from databuilder import Scoped
 from databuilder.extractor.base_extractor import Extractor
 from databuilder.extractor.dashboard.mode_analytics.mode_dashboard_utils import ModeDashboardUtils
-from databuilder.extractor.restapi.rest_api_extractor import MODEL_CLASS
 from databuilder.rest_api.rest_api_query import RestApiQuery
+from databuilder.transformer.base_transformer import ChainedTransformer
+from databuilder.transformer.dict_to_model import DictToModel, MODEL_CLASS
+from databuilder.transformer.timestamp_string_to_epoch import TimestampStringToEpoch, FIELD_NAME
+from databuilder.transformer.template_variable_substitution_transformer import \
+    TemplateVariableSubstitutionTransformer, TEMPLATE, FIELD_NAME as VAR_FIELD_NAME
 
 # CONFIG KEYS
 ORGANIZATION = 'organization'
@@ -36,19 +41,51 @@ class ModeDashboardExtractor(Extractor):
         self._conf = conf
 
         restapi_query = self._build_restapi_query()
-        self._extractor = ModeDashboardUtils.create_mode_rest_api_extractor(
-            restapi_query=restapi_query,
-            conf=self._conf.with_fallback(
+        self._extractor = ModeDashboardUtils.create_mode_rest_api_extractor(restapi_query=restapi_query,
+                                                                            conf=self._conf)
+
+        # Payload from RestApiQuery has timestamp which is ISO8601. Here we are using TimestampStringToEpoch to
+        # transform into epoch and then using DictToModel to convert Dictionary to Model
+        transformers = []
+        timestamp_str_to_epoch_transformer = TimestampStringToEpoch()
+        timestamp_str_to_epoch_transformer.init(
+            conf=Scoped.get_scoped_conf(self._conf, timestamp_str_to_epoch_transformer.get_scope()).with_fallback(
+                ConfigFactory.from_dict({FIELD_NAME: 'created_timestamp', })))
+
+        transformers.append(timestamp_str_to_epoch_transformer)
+
+        dashboard_group_url_transformer = TemplateVariableSubstitutionTransformer()
+        dashboard_group_url_transformer.init(
+            conf=Scoped.get_scoped_conf(self._conf, dashboard_group_url_transformer.get_scope()).with_fallback(
+                ConfigFactory.from_dict({VAR_FIELD_NAME: 'dashboard_group_url',
+                                         TEMPLATE: 'https://app.mode.com/lyft/spaces/{dashboard_group_id}'})))
+
+        transformers.append(dashboard_group_url_transformer)
+
+        dashboard_url_transformer = TemplateVariableSubstitutionTransformer()
+        dashboard_url_transformer.init(
+            conf=Scoped.get_scoped_conf(self._conf, dashboard_url_transformer.get_scope()).with_fallback(
+                ConfigFactory.from_dict({VAR_FIELD_NAME: 'dashboard_url',
+                                         TEMPLATE: 'https://app.mode.com/lyft/reports/{dashboard_id}'})))
+        transformers.append(dashboard_url_transformer)
+
+        dict_to_model_transformer = DictToModel()
+        dict_to_model_transformer.init(
+            conf=Scoped.get_scoped_conf(self._conf, dict_to_model_transformer.get_scope()).with_fallback(
                 ConfigFactory.from_dict(
-                    {MODEL_CLASS: 'databuilder.models.dashboard_metadata.DashboardMetadata', }
-                )
-            )
-        )
+                    {MODEL_CLASS: 'databuilder.models.dashboard.dashboard_metadata.DashboardMetadata'})))
+        transformers.append(dict_to_model_transformer)
+
+        self._transformer = ChainedTransformer(transformers=transformers)
 
     def extract(self):
         # type: () -> Any
 
-        return self._extractor.extract()
+        record = self._extractor.extract()
+        if not record:
+            return None
+
+        return self._transformer.transform(record=record)
 
     def get_scope(self):
         # type: () -> str
@@ -72,8 +109,8 @@ class ModeDashboardExtractor(Extractor):
         # Reports
         # JSONPATH expression. it goes into array which is located in _embedded.reports and then extracts token, name,
         # and description
-        json_path = '_embedded.reports[*].[token,name,description]'
-        field_names = ['dashboard_id', 'dashboard_name', 'description']
+        json_path = '_embedded.reports[*].[token,name,description,created_at]'
+        field_names = ['dashboard_id', 'dashboard_name', 'description', 'created_timestamp']
         reports_query = RestApiQuery(query_to_join=spaces_query, url=reports_url_template, params=params,
                                      json_path=json_path, field_names=field_names, skip_no_result=True)
         return reports_query

databuilder/extractor/dashboard/mode_analytics/mode_dashboard_last_modified_timestamp_extractor.py

+import logging
+
+from pyhocon import ConfigTree, ConfigFactory  # noqa: F401
+
+from databuilder.extractor.dashboard.mode_analytics.mode_dashboard_executions_extractor import \
+    ModeDashboardExecutionsExtractor
+from databuilder.extractor.dashboard.mode_analytics.mode_dashboard_utils import ModeDashboardUtils
+from databuilder.extractor.restapi.rest_api_extractor import STATIC_RECORD_DICT
+from databuilder.rest_api.rest_api_query import RestApiQuery
+from databuilder.transformer.dict_to_model import DictToModel, MODEL_CLASS
+from databuilder.transformer.timestamp_string_to_epoch import TimestampStringToEpoch, FIELD_NAME
+
+LOGGER = logging.getLogger(__name__)
+
+
+class ModeDashboardLastModifiedTimestampExtractor(ModeDashboardExecutionsExtractor):
+    """
+    A Extractor that extracts Mode dashboard's last modified timestamp.
+
+    """
+
+    def __init__(self):
+        super(ModeDashboardLastModifiedTimestampExtractor, self).__init__()
+
+    def init(self, conf):
+        # type: (ConfigTree) -> None
+
+        conf = conf.with_fallback(
+            ConfigFactory.from_dict({
+                STATIC_RECORD_DICT: {'product': 'mode'},
+                '{}.{}'.format(DictToModel().get_scope(), MODEL_CLASS):
+                    'databuilder.models.dashboard.dashboard_last_modified.DashboardLastModifiedTimestamp',
+                '{}.{}'.format(TimestampStringToEpoch().get_scope(), FIELD_NAME):
+                    'last_modified_timestamp'
+            })
+        )
+        super(ModeDashboardLastModifiedTimestampExtractor, self).init(conf)
+
+    def get_scope(self):
+        # type: () -> str
+        return 'extractor.mode_dashboard_last_modified_timestamp_execution'
+
+    def _build_restapi_query(self):
+        """
+        Build REST API Query. To get Mode Dashboard last modified timestamp, it needs to call two APIs (spaces API,
+        and reports API) joining together.
+        :return: A RestApiQuery that provides Mode Dashboard last successful execution (run)
+        """
+        # type: () -> RestApiQuery
+
+        spaces_query = ModeDashboardUtils.get_spaces_query_api(conf=self._conf)
+        params = ModeDashboardUtils.get_auth_params(conf=self._conf)
+
+        # Reports
+        # https://mode.com/developer/api-reference/analytics/reports/#listReportsInSpace
+        url = 'https://app.mode.com/api/{organization}/spaces/{dashboard_group_id}/reports'
+        json_path = '_embedded.reports[*].[token,edited_at]'
+        field_names = ['dashboard_id', 'last_modified_timestamp']
+        last_modified_query = RestApiQuery(query_to_join=spaces_query, url=url, params=params,
+                                           json_path=json_path, field_names=field_names, skip_no_result=True)
+
+        return last_modified_query

databuilder/models/cluster/cluster_constants.py

+CLUSTER_NODE_LABEL = 'Cluster'
+
+CLUSTER_RELATION_TYPE = 'CLUSTER'
+CLUSTER_REVERSE_RELATION_TYPE = 'CLUSTER_OF'
+
+CLUSTER_NAME_PROP_KEY = 'name'

databuilder/models/dashboard/dashboard_execution.py

@@ -2,7 +2,7 @@ import logging
 
 from typing import Optional, Dict, Any, Union, Iterator  # noqa: F401
 
-from databuilder.models.dashboard_metadata import DashboardMetadata
+from databuilder.models.dashboard.dashboard_metadata import DashboardMetadata
 from databuilder.models.neo4j_csv_serde import (
     Neo4jCsvSerializable, NODE_LABEL, NODE_KEY, RELATION_START_KEY, RELATION_END_KEY, RELATION_START_LABEL,
     RELATION_END_LABEL, RELATION_TYPE, RELATION_REVERSE_TYPE)
@@ -17,8 +17,8 @@ class DashboardExecution(Neo4jCsvSerializable):
     DASHBOARD_EXECUTION_LABEL = 'Execution'
     DASHBOARD_EXECUTION_KEY_FORMAT = '{product}_dashboard://{cluster}.{dashboard_group_id}/' \
                                      '{dashboard_id}/execution/{execution_id}'
-    DASHBOARD_EXECUTION_RELATION_TYPE = 'LAST_EXECUTED'
-    EXECUTION_DASHBOARD_RELATION_TYPE = 'LAST_EXECUTION_OF'
+    DASHBOARD_EXECUTION_RELATION_TYPE = 'EXECUTED'
+    EXECUTION_DASHBOARD_RELATION_TYPE = 'EXECUTION_OF'
 
     LAST_EXECUTION_ID = '_last_execution'
     LAST_SUCCESSFUL_EXECUTION_ID = '_last_successful_execution'
@@ -55,7 +55,7 @@ class DashboardExecution(Neo4jCsvSerializable):
         yield {
             NODE_LABEL: DashboardExecution.DASHBOARD_EXECUTION_LABEL,
             NODE_KEY: self._get_last_execution_node_key(),
-            'time_stamp': self._execution_timestamp,
+            'timestamp': self._execution_timestamp,
             'state': self._execution_state
         }
 

databuilder/models/dashboard/dashboard_last_modified.py

+import logging
+
+from typing import Optional, Dict, Any, Union, Iterator  # noqa: F401
+
+from databuilder.models.dashboard.dashboard_metadata import DashboardMetadata
+from databuilder.models.neo4j_csv_serde import (
+    Neo4jCsvSerializable, NODE_LABEL, NODE_KEY, RELATION_START_KEY, RELATION_END_KEY, RELATION_START_LABEL,
+    RELATION_END_LABEL, RELATION_TYPE, RELATION_REVERSE_TYPE)
+from databuilder.models.timestamp import timestamp_constants
+
+LOGGER = logging.getLogger(__name__)
+
+
+class DashboardLastModifiedTimestamp(Neo4jCsvSerializable):
+    """
+    A model that encapsulate Dashboard's last modified timestamp in epoch
+    """
+
+    DASHBOARD_LAST_MODIFIED_KEY_FORMAT = '{product}_dashboard://{cluster}.{dashboard_group_id}/' \
+                                         '{dashboard_id}/_last_modified_timestamp'
+
+    def __init__(self,
+                 dashboard_group_id,  # type: Optional[str]
+                 dashboard_id,  # type: Optional[str]
+                 last_modified_timestamp,  # type: int
+                 product='',  # type: Optional[str]
+                 cluster='gold',  # type: str
+                 **kwargs
+                 ):
+        self._dashboard_group_id = dashboard_group_id
+        self._dashboard_id = dashboard_id
+        self._last_modified_timestamp = last_modified_timestamp
+        self._product = product
+        self._cluster = cluster
+        self._node_iterator = self._create_node_iterator()
+        self._relation_iterator = self._create_relation_iterator()
+
+    def create_next_node(self):
+        # type: () -> Union[Dict[str, Any], None]
+        try:
+            return next(self._node_iterator)
+        except StopIteration:
+            return None
+
+    def _create_node_iterator(self):  # noqa: C901
+        # type: () -> Iterator[[Dict[str, Any]]]
+        yield {
+            NODE_LABEL: timestamp_constants.NODE_LABEL,
+            NODE_KEY: self._get_last_modified_node_key(),
+            timestamp_constants.TIMESTAMP_PROPERTY: self._last_modified_timestamp,
+            timestamp_constants.TIMESTAMP_NAME_PROPERTY: timestamp_constants.TimestampName.last_updated_timestamp.name,
+        }
+
+    def create_next_relation(self):
+        # type: () -> Union[Dict[str, Any], None]
+        try:
+            return next(self._relation_iterator)
+        except StopIteration:
+            return None
+
+    def _create_relation_iterator(self):
+        # type: () -> Iterator[[Dict[str, Any]]]
+        yield {
+            RELATION_START_LABEL: DashboardMetadata.DASHBOARD_NODE_LABEL,
+            RELATION_END_LABEL: timestamp_constants.NODE_LABEL,
+            RELATION_START_KEY: DashboardMetadata.DASHBOARD_KEY_FORMAT.format(
+                product=self._product,
+                cluster=self._cluster,
+                dashboard_group=self._dashboard_group_id,
+                dashboard_name=self._dashboard_id
+            ),
+            RELATION_END_KEY: self._get_last_modified_node_key(),
+            RELATION_TYPE: timestamp_constants.LASTUPDATED_RELATION_TYPE,
+            RELATION_REVERSE_TYPE: timestamp_constants.LASTUPDATED_REVERSE_RELATION_TYPE
+        }
+
+    def _get_last_modified_node_key(self):
+        return DashboardLastModifiedTimestamp.DASHBOARD_LAST_MODIFIED_KEY_FORMAT.format(
+            product=self._product,
+            cluster=self._cluster,
+            dashboard_group_id=self._dashboard_group_id,
+            dashboard_id=self._dashboard_id,
+        )
+
+    def __repr__(self):
+        return 'DashboardLastModifiedTimestamp({!r}, {!r}, {!r}, {!r}, {!r})'.format(
+            self._dashboard_group_id,
+            self._dashboard_id,
+            self._last_modified_timestamp,
+            self._product,
+            self._cluster
+        )

databuilder/models/dashboard/dashboard_owner.py

@@ -2,7 +2,7 @@ import logging
 
 from typing import Optional, Dict, Any, Union, Iterator  # noqa: F401
 
-from databuilder.models.dashboard_metadata import DashboardMetadata
+from databuilder.models.dashboard.dashboard_metadata import DashboardMetadata
 from databuilder.models.neo4j_csv_serde import (
     Neo4jCsvSerializable, RELATION_START_KEY, RELATION_END_KEY, RELATION_START_LABEL,
     RELATION_END_LABEL, RELATION_TYPE, RELATION_REVERSE_TYPE)

databuilder/models/table_last_updated.py

@@ -5,17 +5,18 @@ from databuilder.models.neo4j_csv_serde import Neo4jCsvSerializable, NODE_KEY, \
     RELATION_END_LABEL, RELATION_TYPE, RELATION_REVERSE_TYPE
 
 from databuilder.models.table_metadata import TableMetadata
+from databuilder.models.timestamp import timestamp_constants
 
 
 class TableLastUpdated(Neo4jCsvSerializable):
     # constants
-    LAST_UPDATED_NODE_LABEL = 'Timestamp'
+    LAST_UPDATED_NODE_LABEL = timestamp_constants.NODE_LABEL
     LAST_UPDATED_KEY_FORMAT = '{db}://{cluster}.{schema}/{tbl}/timestamp'
-    TIMESTAMP_PROPERTY = 'last_updated_timestamp'
-    TIMESTAMP_NAME_PROPERTY = 'name'
+    TIMESTAMP_PROPERTY = timestamp_constants.DEPRECATED_TIMESTAMP_PROPERTY
+    TIMESTAMP_NAME_PROPERTY = timestamp_constants.TIMESTAMP_NAME_PROPERTY
 
-    TABLE_LASTUPDATED_RELATION_TYPE = 'LAST_UPDATED_AT'
-    LASTUPDATED_TABLE_RELATION_TYPE = 'LAST_UPDATED_TIME_OF'
+    TABLE_LASTUPDATED_RELATION_TYPE = timestamp_constants.LASTUPDATED_RELATION_TYPE
+    LASTUPDATED_TABLE_RELATION_TYPE = timestamp_constants.LASTUPDATED_REVERSE_RELATION_TYPE
 
     def __init__(self,
                  table_name,  # type: str
@@ -83,7 +84,8 @@ class TableLastUpdated(Neo4jCsvSerializable):
             NODE_KEY: self.get_last_updated_model_key(),
             NODE_LABEL: TableLastUpdated.LAST_UPDATED_NODE_LABEL,
             TableLastUpdated.TIMESTAMP_PROPERTY: self.last_updated_time,
-            TableLastUpdated.TIMESTAMP_NAME_PROPERTY: TableLastUpdated.TIMESTAMP_PROPERTY
+            timestamp_constants.TIMESTAMP_PROPERTY: self.last_updated_time,
+            TableLastUpdated.TIMESTAMP_NAME_PROPERTY: timestamp_constants.TimestampName.last_updated_timestamp.name
         })
 
         return results

databuilder/models/table_metadata.py

@@ -3,12 +3,12 @@ from collections import namedtuple
 
 from typing import Iterable, Any, Union, Iterator, Dict, Set  # noqa: F401
 
+from databuilder.models.cluster import cluster_constants
 from databuilder.models.neo4j_csv_serde import (
     Neo4jCsvSerializable, NODE_LABEL, NODE_KEY, RELATION_START_KEY, RELATION_END_KEY, RELATION_START_LABEL,
     RELATION_END_LABEL, RELATION_TYPE, RELATION_REVERSE_TYPE)
 from databuilder.publisher.neo4j_csv_publisher import UNQUOTED_SUFFIX
 
-
 DESCRIPTION_NODE_LABEL_VAL = 'Description'
 DESCRIPTION_NODE_LABEL = DESCRIPTION_NODE_LABEL_VAL
 
@@ -203,10 +203,10 @@ class TableMetadata(Neo4jCsvSerializable):
 
     DATABASE_NODE_LABEL = 'Database'
     DATABASE_KEY_FORMAT = 'database://{db}'
-    DATABASE_CLUSTER_RELATION_TYPE = 'CLUSTER'
-    CLUSTER_DATABASE_RELATION_TYPE = 'CLUSTER_OF'
+    DATABASE_CLUSTER_RELATION_TYPE = cluster_constants.CLUSTER_RELATION_TYPE
+    CLUSTER_DATABASE_RELATION_TYPE = cluster_constants.CLUSTER_REVERSE_RELATION_TYPE
 
-    CLUSTER_NODE_LABEL = 'Cluster'
+    CLUSTER_NODE_LABEL = cluster_constants.CLUSTER_NODE_LABEL
     CLUSTER_KEY_FORMAT = '{db}://{cluster}'
     CLUSTER_SCHEMA_RELATION_TYPE = 'SCHEMA'
     SCHEMA_CLUSTER_RELATION_TYPE = 'SCHEMA_OF'
@@ -274,14 +274,14 @@ class TableMetadata(Neo4jCsvSerializable):
     def __repr__(self):
         # type: () -> str
         return 'TableMetadata({!r}, {!r}, {!r}, {!r} ' \
-            '{!r}, {!r}, {!r}, {!r})'.format(self.database,
-                                             self.cluster,
-                                             self.schema,
-                                             self.name,
-                                             self.description,
-                                             self.columns,
-                                             self.is_view,
-                                             self.tags)
+               '{!r}, {!r}, {!r}, {!r})'.format(self.database,
+                                                self.cluster,
+                                                self.schema,
+                                                self.name,
+                                                self.description,
+                                                self.columns,
+                                                self.is_view,
+                                                self.tags)
 
     def _get_table_key(self):
         # type: () -> str

databuilder/models/timestamp/timestamp_constants.py

+from enum import Enum
+
+NODE_LABEL = 'Timestamp'
+
+TIMESTAMP_PROPERTY = 'timestamp'
+TIMESTAMP_NAME_PROPERTY = 'name'
+# This is deprecated property as it's not generic for the Timestamp
+DEPRECATED_TIMESTAMP_PROPERTY = 'last_updated_timestamp'
+
+
+LASTUPDATED_RELATION_TYPE = 'LAST_UPDATED_AT'
+LASTUPDATED_REVERSE_RELATION_TYPE = 'LAST_UPDATED_TIME_OF'
+
+
+class TimestampName(Enum):
+    last_updated_timestamp = 1

databuilder/transformer/template_variable_substitution_transformer.py

+import logging
+
+from pyhocon import ConfigTree  # noqa: F401
+from typing import Any, Dict  # noqa: F401
+
+from databuilder.transformer.base_transformer import Transformer
+
+TEMPLATE = 'template'
+FIELD_NAME = 'field_name'  # field name to UPSERT
+
+LOGGER = logging.getLogger(__name__)
+
+
+class TemplateVariableSubstitutionTransformer(Transformer):
+    """
+    Transforms dictionary into model
+    """
+
+    def init(self, conf):
+        # type: (ConfigTree) -> None
+
+        self._template = conf.get_string(TEMPLATE)
+        self._field_name = conf.get_string(FIELD_NAME)
+
+    def transform(self, record):
+        # type: (Dict[str, Any]) -> Dict[str, Any]
+
+        val = self._template.format(**record)
+        record[self._field_name] = val
+        return record
+
+    def get_scope(self):
+        # type: () -> str
+        return 'transformer.template_variable_substitution'

tests/unit/extractor/restapi/test_rest_api_extractor.py

@@ -4,7 +4,7 @@ from pyhocon import ConfigFactory  # noqa: F401
 
 from databuilder.extractor.restapi.rest_api_extractor import RestAPIExtractor, REST_API_QUERY, MODEL_CLASS, \
     STATIC_RECORD_DICT
-from databuilder.models.dashboard_metadata import DashboardMetadata
+from databuilder.models.dashboard.dashboard_metadata import DashboardMetadata
 from databuilder.rest_api.base_rest_api_query import RestApiQuerySeed
 
 
@@ -35,7 +35,7 @@ class TestRestAPIExtractor(unittest.TestCase):
                                   'dashboard_name': 'bar',
                                   'description': 'john',
                                   'dashboard_group_description': 'doe'}]),
-                MODEL_CLASS: 'databuilder.models.dashboard_metadata.DashboardMetadata',
+                MODEL_CLASS: 'databuilder.models.dashboard.dashboard_metadata.DashboardMetadata',
             }
         )
         extractor = RestAPIExtractor()

tests/unit/models/dashboard/test_dashboard_last_modified.py

+import unittest
+
+from databuilder.models.dashboard.dashboard_last_modified import DashboardLastModifiedTimestamp
+from databuilder.models.neo4j_csv_serde import RELATION_START_KEY, RELATION_START_LABEL, RELATION_END_KEY, \
+    RELATION_END_LABEL, RELATION_TYPE, RELATION_REVERSE_TYPE
+
+
+class TestDashboardLastModifiedTimestamp(unittest.TestCase):
+
+    def test_dashboard_timestamp_nodes(self):
+        # type: () -> None
+        dashboard_last_modified = DashboardLastModifiedTimestamp(last_modified_timestamp=123456789,
+                                                                 cluster='cluster_id',
+                                                                 product='product_id',
+                                                                 dashboard_id='dashboard_id',
+                                                                 dashboard_group_id='dashboard_group_id')
+
+        actual = dashboard_last_modified.create_next_node()
+        expected = {'timestamp': 123456789,
+                    'name': 'last_updated_timestamp',
+                    'KEY': 'product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id/_last_modified_timestamp',
+                    'LABEL': 'Timestamp'}
+
+        self.assertDictEqual(actual, expected)
+        self.assertIsNone(dashboard_last_modified.create_next_node())
+
+    def test_dashboard_owner_relations(self):
+        # type: () -> None
+        dashboard_last_modified = DashboardLastModifiedTimestamp(last_modified_timestamp=123456789,
+                                                                 cluster='cluster_id',
+                                                                 product='product_id',
+                                                                 dashboard_id='dashboard_id',
+                                                                 dashboard_group_id='dashboard_group_id')
+
+        actual = dashboard_last_modified.create_next_relation()
+        print(actual)
+        expected = {
+            RELATION_END_KEY: 'product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id'
+                              '/_last_modified_timestamp',
+            RELATION_START_LABEL: 'Dashboard',
+            RELATION_END_LABEL: 'Timestamp',
+            RELATION_START_KEY: 'product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id',
+            RELATION_TYPE: 'LAST_UPDATED_AT',
+            RELATION_REVERSE_TYPE: 'LAST_UPDATED_TIME_OF'
+        }
+
+        self.assertDictEqual(actual, expected)
+        self.assertIsNone(dashboard_last_modified.create_next_relation())

tests/unit/models/test_table_last_updated.py

@@ -4,6 +4,7 @@ from databuilder.models.neo4j_csv_serde import NODE_KEY, \
     NODE_LABEL, RELATION_START_KEY, RELATION_START_LABEL, RELATION_END_KEY, \
     RELATION_END_LABEL, RELATION_TYPE, RELATION_REVERSE_TYPE
 from databuilder.models.table_last_updated import TableLastUpdated
+from databuilder.models.timestamp import timestamp_constants
 
 
 class TestTableLastUpdated(unittest.TestCase):
@@ -20,6 +21,7 @@ class TestTableLastUpdated(unittest.TestCase):
             NODE_KEY: 'hive://gold.default/test_table/timestamp',
             NODE_LABEL: 'Timestamp',
             'last_updated_timestamp': 25195665,
+            timestamp_constants.TIMESTAMP_PROPERTY: 25195665,
             'name': 'last_updated_timestamp'
         }
 
@@ -35,12 +37,12 @@ class TestTableLastUpdated(unittest.TestCase):
     def test_create_next_node(self):
         # type: () -> None
         next_node = self.tableLastUpdated.create_next_node()
-        self.assertEquals(next_node, self.expected_node_result)
+        self.assertEqual(next_node, self.expected_node_result)
 
     def test_create_next_relation(self):
         # type: () -> None
         next_relation = self.tableLastUpdated.create_next_relation()
-        self.assertEquals(next_relation, self.expected_relation_result)
+        self.assertEqual(next_relation, self.expected_relation_result)
 
     def test_get_table_model_key(self):
         # type: () -> None

tests/unit/transformer/test_template_variable_substitution_transformer.py

+import unittest
+
+from pyhocon import ConfigFactory
+
+from databuilder.transformer.template_variable_substitution_transformer import \
+    TemplateVariableSubstitutionTransformer, FIELD_NAME, TEMPLATE
+
+
+class TestTemplateVariableSubstitutionTransformer(unittest.TestCase):
+
+    def test_conversion(self):
+        # type: () -> None
+
+        transformer = TemplateVariableSubstitutionTransformer()
+        config = ConfigFactory.from_dict({
+            FIELD_NAME: 'baz',
+            TEMPLATE: 'Hello {foo}'
+        })
+        transformer.init(conf=config)
+
+        actual = transformer.transform({'foo': 'bar'})
+        expected = {
+            'foo': 'bar',
+            'baz': 'Hello bar'
+        }
+        self.assertDictEqual(expected, actual)
+
+
+if __name__ == '__main__':
+    unittest.main()