Added created timestamp and last modified timestamp on Dashboard (#207)

* Added created timestamp and last modified timestamp on Dashboard

* Remove owner and reload time from dashboard_metadata

* Move dashboard_metadata under dashboard package

* Update

* Update

* Added dashboard group url and dashboard url

* Added cluster node in Dashboard

databuilder/extractor/dashboard/mode_analytics/mode_dashboard_extractor.py

@@ -3,10 +3,15 @@ import logging
 from pyhocon import ConfigTree, ConfigFactory  # noqa: F401
 from typing import Any  # noqa: F401
 
+from databuilder import Scoped
 from databuilder.extractor.base_extractor import Extractor
 from databuilder.extractor.dashboard.mode_analytics.mode_dashboard_utils import ModeDashboardUtils
-from databuilder.extractor.restapi.rest_api_extractor import MODEL_CLASS
 from databuilder.rest_api.rest_api_query import RestApiQuery
+from databuilder.transformer.base_transformer import ChainedTransformer
+from databuilder.transformer.dict_to_model import DictToModel, MODEL_CLASS
+from databuilder.transformer.timestamp_string_to_epoch import TimestampStringToEpoch, FIELD_NAME
+from databuilder.transformer.template_variable_substitution_transformer import \
+    TemplateVariableSubstitutionTransformer, TEMPLATE, FIELD_NAME as VAR_FIELD_NAME
 
 # CONFIG KEYS
 ORGANIZATION = 'organization'
@@ -36,19 +41,51 @@ class ModeDashboardExtractor(Extractor):
         self._conf = conf
 
         restapi_query = self._build_restapi_query()
-        self._extractor = ModeDashboardUtils.create_mode_rest_api_extractor(
-            restapi_query=restapi_query,
-            conf=self._conf.with_fallback(
+        self._extractor = ModeDashboardUtils.create_mode_rest_api_extractor(restapi_query=restapi_query,
+                                                                            conf=self._conf)
+
+        # Payload from RestApiQuery has timestamp which is ISO8601. Here we are using TimestampStringToEpoch to
+        # transform into epoch and then using DictToModel to convert Dictionary to Model
+        transformers = []
+        timestamp_str_to_epoch_transformer = TimestampStringToEpoch()
+        timestamp_str_to_epoch_transformer.init(
+            conf=Scoped.get_scoped_conf(self._conf, timestamp_str_to_epoch_transformer.get_scope()).with_fallback(
+                ConfigFactory.from_dict({FIELD_NAME: 'created_timestamp', })))
+
+        transformers.append(timestamp_str_to_epoch_transformer)
+
+        dashboard_group_url_transformer = TemplateVariableSubstitutionTransformer()
+        dashboard_group_url_transformer.init(
+            conf=Scoped.get_scoped_conf(self._conf, dashboard_group_url_transformer.get_scope()).with_fallback(
+                ConfigFactory.from_dict({VAR_FIELD_NAME: 'dashboard_group_url',
+                                         TEMPLATE: 'https://app.mode.com/lyft/spaces/{dashboard_group_id}'})))
+
+        transformers.append(dashboard_group_url_transformer)
+
+        dashboard_url_transformer = TemplateVariableSubstitutionTransformer()
+        dashboard_url_transformer.init(
+            conf=Scoped.get_scoped_conf(self._conf, dashboard_url_transformer.get_scope()).with_fallback(
+                ConfigFactory.from_dict({VAR_FIELD_NAME: 'dashboard_url',
+                                         TEMPLATE: 'https://app.mode.com/lyft/reports/{dashboard_id}'})))
+        transformers.append(dashboard_url_transformer)
+
+        dict_to_model_transformer = DictToModel()
+        dict_to_model_transformer.init(
+            conf=Scoped.get_scoped_conf(self._conf, dict_to_model_transformer.get_scope()).with_fallback(
                 ConfigFactory.from_dict(
-                    {MODEL_CLASS: 'databuilder.models.dashboard_metadata.DashboardMetadata', }
-                )
-            )
-        )
+                    {MODEL_CLASS: 'databuilder.models.dashboard.dashboard_metadata.DashboardMetadata'})))
+        transformers.append(dict_to_model_transformer)
+
+        self._transformer = ChainedTransformer(transformers=transformers)
 
     def extract(self):
         # type: () -> Any
 
-        return self._extractor.extract()
+        record = self._extractor.extract()
+        if not record:
+            return None
+
+        return self._transformer.transform(record=record)
 
     def get_scope(self):
         # type: () -> str
@@ -72,8 +109,8 @@ class ModeDashboardExtractor(Extractor):
         # Reports
         # JSONPATH expression. it goes into array which is located in _embedded.reports and then extracts token, name,
         # and description
-        json_path = '_embedded.reports[*].[token,name,description]'
-        field_names = ['dashboard_id', 'dashboard_name', 'description']
+        json_path = '_embedded.reports[*].[token,name,description,created_at]'
+        field_names = ['dashboard_id', 'dashboard_name', 'description', 'created_timestamp']
         reports_query = RestApiQuery(query_to_join=spaces_query, url=reports_url_template, params=params,
                                      json_path=json_path, field_names=field_names, skip_no_result=True)
         return reports_query

databuilder/extractor/dashboard/mode_analytics/mode_dashboard_last_modified_timestamp_extractor.py

+import logging
+
+from pyhocon import ConfigTree, ConfigFactory  # noqa: F401
+
+from databuilder.extractor.dashboard.mode_analytics.mode_dashboard_executions_extractor import \
+    ModeDashboardExecutionsExtractor
+from databuilder.extractor.dashboard.mode_analytics.mode_dashboard_utils import ModeDashboardUtils
+from databuilder.extractor.restapi.rest_api_extractor import STATIC_RECORD_DICT
+from databuilder.rest_api.rest_api_query import RestApiQuery
+from databuilder.transformer.dict_to_model import DictToModel, MODEL_CLASS
+from databuilder.transformer.timestamp_string_to_epoch import TimestampStringToEpoch, FIELD_NAME
+
+LOGGER = logging.getLogger(__name__)
+
+
+class ModeDashboardLastModifiedTimestampExtractor(ModeDashboardExecutionsExtractor):
+    """
+    A Extractor that extracts Mode dashboard's last modified timestamp.
+
+    """
+
+    def __init__(self):
+        super(ModeDashboardLastModifiedTimestampExtractor, self).__init__()
+
+    def init(self, conf):
+        # type: (ConfigTree) -> None
+
+        conf = conf.with_fallback(
+            ConfigFactory.from_dict({
+                STATIC_RECORD_DICT: {'product': 'mode'},
+                '{}.{}'.format(DictToModel().get_scope(), MODEL_CLASS):
+                    'databuilder.models.dashboard.dashboard_last_modified.DashboardLastModifiedTimestamp',
+                '{}.{}'.format(TimestampStringToEpoch().get_scope(), FIELD_NAME):
+                    'last_modified_timestamp'
+            })
+        )
+        super(ModeDashboardLastModifiedTimestampExtractor, self).init(conf)
+
+    def get_scope(self):
+        # type: () -> str
+        return 'extractor.mode_dashboard_last_modified_timestamp_execution'
+
+    def _build_restapi_query(self):
+        """
+        Build REST API Query. To get Mode Dashboard last modified timestamp, it needs to call two APIs (spaces API,
+        and reports API) joining together.
+        :return: A RestApiQuery that provides Mode Dashboard last successful execution (run)
+        """
+        # type: () -> RestApiQuery
+
+        spaces_query = ModeDashboardUtils.get_spaces_query_api(conf=self._conf)
+        params = ModeDashboardUtils.get_auth_params(conf=self._conf)
+
+        # Reports
+        # https://mode.com/developer/api-reference/analytics/reports/#listReportsInSpace
+        url = 'https://app.mode.com/api/{organization}/spaces/{dashboard_group_id}/reports'
+        json_path = '_embedded.reports[*].[token,edited_at]'
+        field_names = ['dashboard_id', 'last_modified_timestamp']
+        last_modified_query = RestApiQuery(query_to_join=spaces_query, url=url, params=params,
+                                           json_path=json_path, field_names=field_names, skip_no_result=True)
+
+        return last_modified_query

databuilder/models/cluster/cluster_constants.py

+CLUSTER_NODE_LABEL = 'Cluster'
+
+CLUSTER_RELATION_TYPE = 'CLUSTER'
+CLUSTER_REVERSE_RELATION_TYPE = 'CLUSTER_OF'
+
+CLUSTER_NAME_PROP_KEY = 'name'

databuilder/models/dashboard/dashboard_execution.py

@@ -2,7 +2,7 @@ import logging
 
 from typing import Optional, Dict, Any, Union, Iterator  # noqa: F401
 
-from databuilder.models.dashboard_metadata import DashboardMetadata
+from databuilder.models.dashboard.dashboard_metadata import DashboardMetadata
 from databuilder.models.neo4j_csv_serde import (
     Neo4jCsvSerializable, NODE_LABEL, NODE_KEY, RELATION_START_KEY, RELATION_END_KEY, RELATION_START_LABEL,
     RELATION_END_LABEL, RELATION_TYPE, RELATION_REVERSE_TYPE)
@@ -17,8 +17,8 @@ class DashboardExecution(Neo4jCsvSerializable):
     DASHBOARD_EXECUTION_LABEL = 'Execution'
     DASHBOARD_EXECUTION_KEY_FORMAT = '{product}_dashboard://{cluster}.{dashboard_group_id}/' \
                                      '{dashboard_id}/execution/{execution_id}'
-    DASHBOARD_EXECUTION_RELATION_TYPE = 'LAST_EXECUTED'
-    EXECUTION_DASHBOARD_RELATION_TYPE = 'LAST_EXECUTION_OF'
+    DASHBOARD_EXECUTION_RELATION_TYPE = 'EXECUTED'
+    EXECUTION_DASHBOARD_RELATION_TYPE = 'EXECUTION_OF'
 
     LAST_EXECUTION_ID = '_last_execution'
     LAST_SUCCESSFUL_EXECUTION_ID = '_last_successful_execution'
@@ -55,7 +55,7 @@ class DashboardExecution(Neo4jCsvSerializable):
         yield {
             NODE_LABEL: DashboardExecution.DASHBOARD_EXECUTION_LABEL,
             NODE_KEY: self._get_last_execution_node_key(),
-            'time_stamp': self._execution_timestamp,
+            'timestamp': self._execution_timestamp,
             'state': self._execution_state
         }
 

databuilder/models/dashboard/dashboard_last_modified.py

+import logging
+
+from typing import Optional, Dict, Any, Union, Iterator  # noqa: F401
+
+from databuilder.models.dashboard.dashboard_metadata import DashboardMetadata
+from databuilder.models.neo4j_csv_serde import (
+    Neo4jCsvSerializable, NODE_LABEL, NODE_KEY, RELATION_START_KEY, RELATION_END_KEY, RELATION_START_LABEL,
+    RELATION_END_LABEL, RELATION_TYPE, RELATION_REVERSE_TYPE)
+from databuilder.models.timestamp import timestamp_constants
+
+LOGGER = logging.getLogger(__name__)
+
+
+class DashboardLastModifiedTimestamp(Neo4jCsvSerializable):
+    """
+    A model that encapsulate Dashboard's last modified timestamp in epoch
+    """
+
+    DASHBOARD_LAST_MODIFIED_KEY_FORMAT = '{product}_dashboard://{cluster}.{dashboard_group_id}/' \
+                                         '{dashboard_id}/_last_modified_timestamp'
+
+    def __init__(self,
+                 dashboard_group_id,  # type: Optional[str]
+                 dashboard_id,  # type: Optional[str]
+                 last_modified_timestamp,  # type: int
+                 product='',  # type: Optional[str]
+                 cluster='gold',  # type: str
+                 **kwargs
+                 ):
+        self._dashboard_group_id = dashboard_group_id
+        self._dashboard_id = dashboard_id
+        self._last_modified_timestamp = last_modified_timestamp
+        self._product = product
+        self._cluster = cluster
+        self._node_iterator = self._create_node_iterator()
+        self._relation_iterator = self._create_relation_iterator()
+
+    def create_next_node(self):
+        # type: () -> Union[Dict[str, Any], None]
+        try:
+            return next(self._node_iterator)
+        except StopIteration:
+            return None
+
+    def _create_node_iterator(self):  # noqa: C901
+        # type: () -> Iterator[[Dict[str, Any]]]
+        yield {
+            NODE_LABEL: timestamp_constants.NODE_LABEL,
+            NODE_KEY: self._get_last_modified_node_key(),
+            timestamp_constants.TIMESTAMP_PROPERTY: self._last_modified_timestamp,
+            timestamp_constants.TIMESTAMP_NAME_PROPERTY: timestamp_constants.TimestampName.last_updated_timestamp.name,
+        }
+
+    def create_next_relation(self):
+        # type: () -> Union[Dict[str, Any], None]
+        try:
+            return next(self._relation_iterator)
+        except StopIteration:
+            return None
+
+    def _create_relation_iterator(self):
+        # type: () -> Iterator[[Dict[str, Any]]]
+        yield {
+            RELATION_START_LABEL: DashboardMetadata.DASHBOARD_NODE_LABEL,
+            RELATION_END_LABEL: timestamp_constants.NODE_LABEL,
+            RELATION_START_KEY: DashboardMetadata.DASHBOARD_KEY_FORMAT.format(
+                product=self._product,
+                cluster=self._cluster,
+                dashboard_group=self._dashboard_group_id,
+                dashboard_name=self._dashboard_id
+            ),
+            RELATION_END_KEY: self._get_last_modified_node_key(),
+            RELATION_TYPE: timestamp_constants.LASTUPDATED_RELATION_TYPE,
+            RELATION_REVERSE_TYPE: timestamp_constants.LASTUPDATED_REVERSE_RELATION_TYPE
+        }
+
+    def _get_last_modified_node_key(self):
+        return DashboardLastModifiedTimestamp.DASHBOARD_LAST_MODIFIED_KEY_FORMAT.format(
+            product=self._product,
+            cluster=self._cluster,
+            dashboard_group_id=self._dashboard_group_id,
+            dashboard_id=self._dashboard_id,
+        )
+
+    def __repr__(self):
+        return 'DashboardLastModifiedTimestamp({!r}, {!r}, {!r}, {!r}, {!r})'.format(
+            self._dashboard_group_id,
+            self._dashboard_id,
+            self._last_modified_timestamp,
+            self._product,
+            self._cluster
+        )

databuilder/models/dashboard_metadata.py → databuilder/models/dashboard/dashboard_metadata.py

@@ -2,12 +2,12 @@ from collections import namedtuple
 
 from typing import Any, Union, Iterator, Dict, Set, Optional  # noqa: F401
 
-# TODO: We could separate TagMetadata from table_metadata to own module
-from databuilder.models.table_metadata import TagMetadata
+from databuilder.models.cluster import cluster_constants
 from databuilder.models.neo4j_csv_serde import (
     Neo4jCsvSerializable, NODE_LABEL, NODE_KEY, RELATION_START_KEY, RELATION_END_KEY, RELATION_START_LABEL,
     RELATION_END_LABEL, RELATION_TYPE, RELATION_REVERSE_TYPE)
-
+# TODO: We could separate TagMetadata from table_metadata to own module
+from databuilder.models.table_metadata import TagMetadata
 
 NodeTuple = namedtuple('KeyName', ['key', 'name', 'label'])
 RelTuple = namedtuple('RelKeys', ['start_label', 'end_label', 'start_key', 'end_key', 'type', 'reverse_type'])
@@ -26,9 +26,16 @@ class DashboardMetadata(Neo4jCsvSerializable):
 
     Lastreloadtime is the time when the Dashboard was last reloaded.
     """
+    CLUSTER_KEY_FORMAT = '{product}_dashboard://{cluster}'
+    CLUSTER_DASHBOARD_GROUP_RELATION_TYPE = 'DASHBOARD_GROUP'
+    DASHBOARD_GROUP_CLUSTER_RELATION_TYPE = 'DASHBOARD_GROUP_OF'
+
     DASHBOARD_NODE_LABEL = 'Dashboard'
     DASHBOARD_KEY_FORMAT = '{product}_dashboard://{cluster}.{dashboard_group}/{dashboard_name}'
     DASHBOARD_NAME = 'name'
+    DASHBOARD_CREATED_TIME_STAMP = 'created_timestamp'
+    DASHBOARD_GROUP_URL = 'dashboard_group_url'
+    DASHBOARD_URL = 'dashboard_url'
 
     DASHBOARD_DESCRIPTION_NODE_LABEL = 'Description'
     DASHBOARD_DESCRIPTION = 'description'
@@ -44,19 +51,6 @@ class DashboardMetadata(Neo4jCsvSerializable):
 
     DASHBOARD_GROUP_DESCRIPTION_KEY_FORMAT = '{product}_dashboard://{cluster}.{dashboard_group}/_description'
 
-    DASHBOARD_LAST_RELOAD_TIME_NODE_LABEL = 'Lastreloadtime'
-    DASHBOARD_LAST_RELOAD_TIME = 'value'
-    DASHBOARD_LAST_RELOAD_TIME_FORMAT =\
-        '{product}_dashboard://{cluster}.{dashboard_group}/{dashboard_name}/_lastreloadtime'
-    DASHBOARD_LAST_RELOAD_TIME_RELATION_TYPE = 'LAST_RELOAD_TIME'
-    LAST_RELOAD_TIME_DASHBOARD_RELATION_TYPE = 'LAST_RELOAD_TIME_OF'
-
-    OWNER_NODE_LABEL = 'User'
-    OWNER_KEY_FORMAT = '{user_id}'
-    DASHBOARD_OWNER_RELATION_TYPE = 'OWNER'
-    OWNER_DASHBOARD_RELATION_TYPE = 'OWNER_OF'
-    OWNER_ID = 'user_id'
-
     DASHBOARD_TAG_RELATION_TYPE = 'TAG'
     TAG_DASHBOARD_RELATION_TYPE = 'TAG_OF'
 
@@ -67,14 +61,15 @@ class DashboardMetadata(Neo4jCsvSerializable):
                  dashboard_group,  # type: str
                  dashboard_name,  # type: str
                  description,  # type: Union[str, None]
-                 last_reload_time=None,  # type: Optional[str]
-                 user_id=None,  # type: Optional[str]
                  tags=None,  # type: List
                  cluster='gold',  # type: str
                  product='',  # type: Optional[str]
                  dashboard_group_id=None,  # type: Optional[str]
                  dashboard_id=None,  # type: Optional[str]
                  dashboard_group_description=None,  # type: Optional[str]
+                 created_timestamp=None,  # type: Optional[int]
+                 dashboard_group_url=None,  # type: Optional[str]
+                 dashboard_url=None,  # type: Optional[str]
                  **kwargs
                  ):
         # type: (...) -> None
@@ -84,29 +79,38 @@ class DashboardMetadata(Neo4jCsvSerializable):
         self.dashboard_group_id = dashboard_group_id if dashboard_group_id else dashboard_group
         self.dashboard_id = dashboard_id if dashboard_id else dashboard_name
         self.description = description
-        self.last_reload_time = last_reload_time
-        self.user_id = user_id
         self.tags = tags
         self.product = product
         self.cluster = cluster
         self.dashboard_group_description = dashboard_group_description
+        self.created_timestamp = created_timestamp
+        self.dashboard_group_url = dashboard_group_url
+        self.dashboard_url = dashboard_url
+        self._processed_cluster = set()
+        self._processed_dashboard_group = set()
         self._node_iterator = self._create_next_node()
         self._relation_iterator = self._create_next_relation()
 
     def __repr__(self):
         # type: () -> str
-        return 'DashboardMetadata({!r}, {!r}, {!r}, {!r}, {!r}, {!r}, {!r}, {!r}, {!r})' \
+        return 'DashboardMetadata({!r}, {!r}, {!r}, {!r}, {!r}, {!r}, {!r}, {!r}, {!r}, {!r})' \
             .format(self.dashboard_group,
                     self.dashboard_name,
                     self.description,
-                    self.last_reload_time,
-                    self.user_id,
                     self.tags,
                     self.dashboard_group_id,
                     self.dashboard_id,
-                    self.dashboard_group_description
+                    self.dashboard_group_description,
+                    self.created_timestamp,
+                    self.dashboard_group_url,
+                    self.dashboard_url,
                     )
 
+    def _get_cluster_key(self):
+        # type: () -> str
+        return DashboardMetadata.CLUSTER_KEY_FORMAT.format(cluster=self.cluster,
+                                                           product=self.product)
+
     def _get_dashboard_key(self):
         # type: () -> str
         return DashboardMetadata.DASHBOARD_KEY_FORMAT.format(dashboard_group=self.dashboard_group_id,
@@ -140,10 +144,6 @@ class DashboardMetadata(Neo4jCsvSerializable):
                                                                           cluster=self.cluster,
                                                                           product=self.product)
 
-    def _get_owner_key(self):
-        # type: () -> str
-        return DashboardMetadata.OWNER_KEY_FORMAT.format(user_id=self.user_id)
-
     def create_next_node(self):
         # type: () -> Union[Dict[str, Any], None]
         try:
@@ -153,18 +153,42 @@ class DashboardMetadata(Neo4jCsvSerializable):
 
     def _create_next_node(self):
         # type: () -> Iterator[Any]
+        # Cluster node
+        if not self._get_cluster_key() in self._processed_cluster:
+            self._processed_cluster.add(self._get_cluster_key())
+            yield {
+                NODE_LABEL: cluster_constants.CLUSTER_NODE_LABEL,
+                NODE_KEY: self._get_cluster_key(),
+                cluster_constants.CLUSTER_NAME_PROP_KEY: self.cluster
+            }
+
         # Dashboard node
-        yield {NODE_LABEL: DashboardMetadata.DASHBOARD_NODE_LABEL,
-               NODE_KEY: self._get_dashboard_key(),
-               DashboardMetadata.DASHBOARD_NAME: self.dashboard_name,
-               }
+        dashboard_node = {
+            NODE_LABEL: DashboardMetadata.DASHBOARD_NODE_LABEL,
+            NODE_KEY: self._get_dashboard_key(),
+            DashboardMetadata.DASHBOARD_NAME: self.dashboard_name,
+        }
+        if self.created_timestamp:
+            dashboard_node[DashboardMetadata.DASHBOARD_CREATED_TIME_STAMP] = self.created_timestamp
+
+        if self.dashboard_url:
+            dashboard_node[DashboardMetadata.DASHBOARD_URL] = self.dashboard_url
+
+        yield dashboard_node
 
         # Dashboard group
-        if self.dashboard_group:
-            yield {NODE_LABEL: DashboardMetadata.DASHBOARD_GROUP_NODE_LABEL,
-                   NODE_KEY: self._get_dashboard_group_key(),
-                   DashboardMetadata.DASHBOARD_NAME: self.dashboard_group,
-                   }
+        if self.dashboard_group and not self._get_dashboard_group_key() in self._processed_dashboard_group:
+            self._processed_dashboard_group.add(self._get_dashboard_group_key())
+            dashboard_group_node = {
+                NODE_LABEL: DashboardMetadata.DASHBOARD_GROUP_NODE_LABEL,
+                NODE_KEY: self._get_dashboard_group_key(),
+                DashboardMetadata.DASHBOARD_NAME: self.dashboard_group,
+            }
+
+            if self.dashboard_group_url:
+                dashboard_group_node[DashboardMetadata.DASHBOARD_GROUP_URL] = self.dashboard_group_url
+
+            yield dashboard_group_node
 
         # Dashboard group description
         if self.dashboard_group_description:
@@ -178,12 +202,6 @@ class DashboardMetadata(Neo4jCsvSerializable):
                    NODE_KEY: self._get_dashboard_description_key(),
                    DashboardMetadata.DASHBOARD_DESCRIPTION: self.description}
 
-        # Dashboard last reload time node
-        if self.last_reload_time:
-            yield {NODE_LABEL: DashboardMetadata.DASHBOARD_LAST_RELOAD_TIME_NODE_LABEL,
-                   NODE_KEY: self._get_dashboard_last_reload_time_key(),
-                   DashboardMetadata.DASHBOARD_LAST_RELOAD_TIME: self.last_reload_time}
-
         # Dashboard tag node
         if self.tags:
             for tag in self.tags:
@@ -201,6 +219,16 @@ class DashboardMetadata(Neo4jCsvSerializable):
     def _create_next_relation(self):
         # type: () -> Iterator[Any]
 
+        # Cluster <-> Dashboard group
+        yield {
+            RELATION_START_LABEL: cluster_constants.CLUSTER_NODE_LABEL,
+            RELATION_END_LABEL: DashboardMetadata.DASHBOARD_GROUP_NODE_LABEL,
+            RELATION_START_KEY: self._get_cluster_key(),
+            RELATION_END_KEY: self._get_dashboard_group_key(),
+            RELATION_TYPE: DashboardMetadata.CLUSTER_DASHBOARD_GROUP_RELATION_TYPE,
+            RELATION_REVERSE_TYPE: DashboardMetadata.DASHBOARD_GROUP_CLUSTER_RELATION_TYPE
+        }
+
         # Dashboard group > Dashboard group description relation
         if self.dashboard_group_description:
             yield {
@@ -233,17 +261,6 @@ class DashboardMetadata(Neo4jCsvSerializable):
                 RELATION_REVERSE_TYPE: DashboardMetadata.DESCRIPTION_DASHBOARD_RELATION_TYPE
             }
 
-        # Dashboard > Dashboard last reload time relation
-        if self.last_reload_time:
-            yield {
-                RELATION_START_LABEL: DashboardMetadata.DASHBOARD_NODE_LABEL,
-                RELATION_END_LABEL: DashboardMetadata.DASHBOARD_LAST_RELOAD_TIME_NODE_LABEL,
-                RELATION_START_KEY: self._get_dashboard_key(),
-                RELATION_END_KEY: self._get_dashboard_last_reload_time_key(),
-                RELATION_TYPE: DashboardMetadata.DASHBOARD_LAST_RELOAD_TIME_RELATION_TYPE,
-                RELATION_REVERSE_TYPE: DashboardMetadata.LAST_RELOAD_TIME_DASHBOARD_RELATION_TYPE
-            }
-
         # Dashboard > Dashboard tag relation
         if self.tags:
             for tag in self.tags:
@@ -255,28 +272,3 @@ class DashboardMetadata(Neo4jCsvSerializable):
                     RELATION_TYPE: DashboardMetadata.DASHBOARD_TAG_RELATION_TYPE,
                     RELATION_REVERSE_TYPE: DashboardMetadata.TAG_DASHBOARD_RELATION_TYPE
                 }
-
-        # Dashboard > Dashboard owner relation
-        others = []
-
-        if self.user_id:
-            others.append(
-                RelTuple(start_label=DashboardMetadata.DASHBOARD_NODE_LABEL,
-                         end_label=DashboardMetadata.OWNER_NODE_LABEL,
-                         start_key=self._get_dashboard_key(),
-                         end_key=self._get_owner_key(),
-                         type=DashboardMetadata.DASHBOARD_OWNER_RELATION_TYPE,
-                         reverse_type=DashboardMetadata.OWNER_DASHBOARD_RELATION_TYPE)
-            )
-
-        for rel_tuple in others:
-            if rel_tuple not in DashboardMetadata.serialized_rels:
-                DashboardMetadata.serialized_rels.add(rel_tuple)
-                yield {
-                    RELATION_START_LABEL: rel_tuple.start_label,
-                    RELATION_END_LABEL: rel_tuple.end_label,
-                    RELATION_START_KEY: rel_tuple.start_key,
-                    RELATION_END_KEY: rel_tuple.end_key,
-                    RELATION_TYPE: rel_tuple.type,
-                    RELATION_REVERSE_TYPE: rel_tuple.reverse_type
-                }

databuilder/models/dashboard/dashboard_owner.py

@@ -2,7 +2,7 @@ import logging
 
 from typing import Optional, Dict, Any, Union, Iterator  # noqa: F401
 
-from databuilder.models.dashboard_metadata import DashboardMetadata
+from databuilder.models.dashboard.dashboard_metadata import DashboardMetadata
 from databuilder.models.neo4j_csv_serde import (
     Neo4jCsvSerializable, RELATION_START_KEY, RELATION_END_KEY, RELATION_START_LABEL,
     RELATION_END_LABEL, RELATION_TYPE, RELATION_REVERSE_TYPE)

databuilder/models/table_last_updated.py

@@ -5,17 +5,18 @@ from databuilder.models.neo4j_csv_serde import Neo4jCsvSerializable, NODE_KEY, \
     RELATION_END_LABEL, RELATION_TYPE, RELATION_REVERSE_TYPE
 
 from databuilder.models.table_metadata import TableMetadata
+from databuilder.models.timestamp import timestamp_constants
 
 
 class TableLastUpdated(Neo4jCsvSerializable):
     # constants
-    LAST_UPDATED_NODE_LABEL = 'Timestamp'
+    LAST_UPDATED_NODE_LABEL = timestamp_constants.NODE_LABEL
     LAST_UPDATED_KEY_FORMAT = '{db}://{cluster}.{schema}/{tbl}/timestamp'
-    TIMESTAMP_PROPERTY = 'last_updated_timestamp'
-    TIMESTAMP_NAME_PROPERTY = 'name'
+    TIMESTAMP_PROPERTY = timestamp_constants.DEPRECATED_TIMESTAMP_PROPERTY
+    TIMESTAMP_NAME_PROPERTY = timestamp_constants.TIMESTAMP_NAME_PROPERTY
 
-    TABLE_LASTUPDATED_RELATION_TYPE = 'LAST_UPDATED_AT'
-    LASTUPDATED_TABLE_RELATION_TYPE = 'LAST_UPDATED_TIME_OF'
+    TABLE_LASTUPDATED_RELATION_TYPE = timestamp_constants.LASTUPDATED_RELATION_TYPE
+    LASTUPDATED_TABLE_RELATION_TYPE = timestamp_constants.LASTUPDATED_REVERSE_RELATION_TYPE
 
     def __init__(self,
                  table_name,  # type: str
@@ -83,7 +84,8 @@ class TableLastUpdated(Neo4jCsvSerializable):
             NODE_KEY: self.get_last_updated_model_key(),
             NODE_LABEL: TableLastUpdated.LAST_UPDATED_NODE_LABEL,
             TableLastUpdated.TIMESTAMP_PROPERTY: self.last_updated_time,
-            TableLastUpdated.TIMESTAMP_NAME_PROPERTY: TableLastUpdated.TIMESTAMP_PROPERTY
+            timestamp_constants.TIMESTAMP_PROPERTY: self.last_updated_time,
+            TableLastUpdated.TIMESTAMP_NAME_PROPERTY: timestamp_constants.TimestampName.last_updated_timestamp.name
         })
 
         return results

databuilder/models/table_metadata.py

@@ -3,12 +3,12 @@ from collections import namedtuple
 
 from typing import Iterable, Any, Union, Iterator, Dict, Set  # noqa: F401
 
+from databuilder.models.cluster import cluster_constants
 from databuilder.models.neo4j_csv_serde import (
     Neo4jCsvSerializable, NODE_LABEL, NODE_KEY, RELATION_START_KEY, RELATION_END_KEY, RELATION_START_LABEL,
     RELATION_END_LABEL, RELATION_TYPE, RELATION_REVERSE_TYPE)
 from databuilder.publisher.neo4j_csv_publisher import UNQUOTED_SUFFIX
 
-
 DESCRIPTION_NODE_LABEL_VAL = 'Description'
 DESCRIPTION_NODE_LABEL = DESCRIPTION_NODE_LABEL_VAL
 
@@ -203,10 +203,10 @@ class TableMetadata(Neo4jCsvSerializable):
 
     DATABASE_NODE_LABEL = 'Database'
     DATABASE_KEY_FORMAT = 'database://{db}'
-    DATABASE_CLUSTER_RELATION_TYPE = 'CLUSTER'
-    CLUSTER_DATABASE_RELATION_TYPE = 'CLUSTER_OF'
+    DATABASE_CLUSTER_RELATION_TYPE = cluster_constants.CLUSTER_RELATION_TYPE
+    CLUSTER_DATABASE_RELATION_TYPE = cluster_constants.CLUSTER_REVERSE_RELATION_TYPE
 
-    CLUSTER_NODE_LABEL = 'Cluster'
+    CLUSTER_NODE_LABEL = cluster_constants.CLUSTER_NODE_LABEL
     CLUSTER_KEY_FORMAT = '{db}://{cluster}'
     CLUSTER_SCHEMA_RELATION_TYPE = 'SCHEMA'
     SCHEMA_CLUSTER_RELATION_TYPE = 'SCHEMA_OF'
@@ -274,14 +274,14 @@ class TableMetadata(Neo4jCsvSerializable):
     def __repr__(self):
         # type: () -> str
         return 'TableMetadata({!r}, {!r}, {!r}, {!r} ' \
-            '{!r}, {!r}, {!r}, {!r})'.format(self.database,
-                                             self.cluster,
-                                             self.schema,
-                                             self.name,
-                                             self.description,
-                                             self.columns,
-                                             self.is_view,
-                                             self.tags)
+               '{!r}, {!r}, {!r}, {!r})'.format(self.database,
+                                                self.cluster,
+                                                self.schema,
+                                                self.name,
+                                                self.description,
+                                                self.columns,
+                                                self.is_view,
+                                                self.tags)
 
     def _get_table_key(self):
         # type: () -> str

databuilder/models/timestamp/timestamp_constants.py

+from enum import Enum
+
+NODE_LABEL = 'Timestamp'
+
+TIMESTAMP_PROPERTY = 'timestamp'
+TIMESTAMP_NAME_PROPERTY = 'name'
+# This is deprecated property as it's not generic for the Timestamp
+DEPRECATED_TIMESTAMP_PROPERTY = 'last_updated_timestamp'
+
+
+LASTUPDATED_RELATION_TYPE = 'LAST_UPDATED_AT'
+LASTUPDATED_REVERSE_RELATION_TYPE = 'LAST_UPDATED_TIME_OF'
+
+
+class TimestampName(Enum):
+    last_updated_timestamp = 1

databuilder/transformer/template_variable_substitution_transformer.py

+import logging
+
+from pyhocon import ConfigTree  # noqa: F401
+from typing import Any, Dict  # noqa: F401
+
+from databuilder.transformer.base_transformer import Transformer
+
+TEMPLATE = 'template'
+FIELD_NAME = 'field_name'  # field name to UPSERT
+
+LOGGER = logging.getLogger(__name__)
+
+
+class TemplateVariableSubstitutionTransformer(Transformer):
+    """
+    Transforms dictionary into model
+    """
+
+    def init(self, conf):
+        # type: (ConfigTree) -> None
+
+        self._template = conf.get_string(TEMPLATE)
+        self._field_name = conf.get_string(FIELD_NAME)
+
+    def transform(self, record):
+        # type: (Dict[str, Any]) -> Dict[str, Any]
+
+        val = self._template.format(**record)
+        record[self._field_name] = val
+        return record
+
+    def get_scope(self):
+        # type: () -> str
+        return 'transformer.template_variable_substitution'

tests/unit/extractor/restapi/test_rest_api_extractor.py

@@ -4,7 +4,7 @@ from pyhocon import ConfigFactory  # noqa: F401
 
 from databuilder.extractor.restapi.rest_api_extractor import RestAPIExtractor, REST_API_QUERY, MODEL_CLASS, \
     STATIC_RECORD_DICT
-from databuilder.models.dashboard_metadata import DashboardMetadata
+from databuilder.models.dashboard.dashboard_metadata import DashboardMetadata
 from databuilder.rest_api.base_rest_api_query import RestApiQuerySeed
 
 
@@ -35,7 +35,7 @@ class TestRestAPIExtractor(unittest.TestCase):
                                   'dashboard_name': 'bar',
                                   'description': 'john',
                                   'dashboard_group_description': 'doe'}]),
-                MODEL_CLASS: 'databuilder.models.dashboard_metadata.DashboardMetadata',
+                MODEL_CLASS: 'databuilder.models.dashboard.dashboard_metadata.DashboardMetadata',
             }
         )
         extractor = RestAPIExtractor()

tests/unit/models/dashboard/test_dashboard_last_modified.py

+import unittest
+
+from databuilder.models.dashboard.dashboard_last_modified import DashboardLastModifiedTimestamp
+from databuilder.models.neo4j_csv_serde import RELATION_START_KEY, RELATION_START_LABEL, RELATION_END_KEY, \
+    RELATION_END_LABEL, RELATION_TYPE, RELATION_REVERSE_TYPE
+
+
+class TestDashboardLastModifiedTimestamp(unittest.TestCase):
+
+    def test_dashboard_timestamp_nodes(self):
+        # type: () -> None
+        dashboard_last_modified = DashboardLastModifiedTimestamp(last_modified_timestamp=123456789,
+                                                                 cluster='cluster_id',
+                                                                 product='product_id',
+                                                                 dashboard_id='dashboard_id',
+                                                                 dashboard_group_id='dashboard_group_id')
+
+        actual = dashboard_last_modified.create_next_node()
+        expected = {'timestamp': 123456789,
+                    'name': 'last_updated_timestamp',
+                    'KEY': 'product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id/_last_modified_timestamp',
+                    'LABEL': 'Timestamp'}
+
+        self.assertDictEqual(actual, expected)
+        self.assertIsNone(dashboard_last_modified.create_next_node())
+
+    def test_dashboard_owner_relations(self):
+        # type: () -> None
+        dashboard_last_modified = DashboardLastModifiedTimestamp(last_modified_timestamp=123456789,
+                                                                 cluster='cluster_id',
+                                                                 product='product_id',
+                                                                 dashboard_id='dashboard_id',
+                                                                 dashboard_group_id='dashboard_group_id')
+
+        actual = dashboard_last_modified.create_next_relation()
+        print(actual)
+        expected = {
+            RELATION_END_KEY: 'product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id'
+                              '/_last_modified_timestamp',
+            RELATION_START_LABEL: 'Dashboard',
+            RELATION_END_LABEL: 'Timestamp',
+            RELATION_START_KEY: 'product_id_dashboard://cluster_id.dashboard_group_id/dashboard_id',
+            RELATION_TYPE: 'LAST_UPDATED_AT',
+            RELATION_REVERSE_TYPE: 'LAST_UPDATED_TIME_OF'
+        }
+
+        self.assertDictEqual(actual, expected)
+        self.assertIsNone(dashboard_last_modified.create_next_relation())

tests/unit/models/test_dashboard_metadata.py → tests/unit/models/dashboard/test_dashboard_metadata.py

 import copy
 import unittest
 
-from databuilder.models.dashboard_metadata import DashboardMetadata
+from databuilder.models.dashboard.dashboard_metadata import DashboardMetadata
 
 
 class TestDashboardMetadata(unittest.TestCase):
@@ -11,26 +11,23 @@ class TestDashboardMetadata(unittest.TestCase):
         self.dashboard_metadata = DashboardMetadata('Product - Jobs.cz',
                                                     'Agent',
                                                     'Agent dashboard description',
-                                                    '2019-05-30T07:03:35.580Z',
-                                                    'roald.amundsen@example.org',
                                                     ['test_tag', 'tag2'],
-                                                    dashboard_group_description='foo dashboard group description'
+                                                    dashboard_group_description='foo dashboard group description',
+                                                    created_timestamp=123456789,
+                                                    dashboard_group_url='https://foo.bar/dashboard_group/foo',
+                                                    dashboard_url='https://foo.bar/dashboard_group/foo/dashboard/bar',
                                                     )
         # Without tags
         self.dashboard_metadata2 = DashboardMetadata('Product - Atmoskop',
                                                      'Atmoskop',
                                                      'Atmoskop dashboard description',
-                                                     '2019-05-30T07:07:42.326Z',
-                                                     'buzz@example.org',
-                                                     []
+                                                     [],
                                                      )
 
         # One common tag with dashboard_metadata, no description
         self.dashboard_metadata3 = DashboardMetadata('Product - Jobs.cz',
                                                      'Dohazovac',
                                                      '',
-                                                     '2019-05-30T07:07:42.326Z',
-                                                     'buzz@example.org',
                                                      ['test_tag', 'tag3']
                                                      )
 
@@ -38,20 +35,20 @@ class TestDashboardMetadata(unittest.TestCase):
         self.dashboard_metadata4 = DashboardMetadata('',
                                                      'PzR',
                                                      '',
-                                                     '2019-05-30T07:07:42.326Z',
-                                                     '',
                                                      []
                                                      )
 
         self.expected_nodes_deduped = [
-            {'name': 'Agent', 'KEY': '_dashboard://gold.Product - Jobs.cz/Agent', 'LABEL': 'Dashboard'},
-            {'name': 'Product - Jobs.cz', 'KEY': '_dashboard://gold.Product - Jobs.cz', 'LABEL': 'Dashboardgroup'},
+            {'KEY': '_dashboard://gold', 'LABEL': 'Cluster', 'name': 'gold'},
+            {'created_timestamp': 123456789, 'name': 'Agent', 'KEY': '_dashboard://gold.Product - Jobs.cz/Agent',
+             'LABEL': 'Dashboard',
+             'dashboard_url': 'https://foo.bar/dashboard_group/foo/dashboard/bar'},
+            {'name': 'Product - Jobs.cz', 'KEY': '_dashboard://gold.Product - Jobs.cz', 'LABEL': 'Dashboardgroup',
+             'dashboard_group_url': 'https://foo.bar/dashboard_group/foo'},
             {'KEY': '_dashboard://gold.Product - Jobs.cz/_description', 'LABEL': 'Description',
              'description': 'foo dashboard group description'},
             {'description': 'Agent dashboard description',
              'KEY': '_dashboard://gold.Product - Jobs.cz/Agent/_description', 'LABEL': 'Description'},
-            {'value': '2019-05-30T07:03:35.580Z',
-             'KEY': '_dashboard://gold.Product - Jobs.cz/Agent/_lastreloadtime', 'LABEL': 'Lastreloadtime'},
             {'tag_type': 'dashboard', 'KEY': 'test_tag', 'LABEL': 'Tag'},
             {'tag_type': 'dashboard', 'KEY': 'tag2', 'LABEL': 'Tag'}
         ]
@@ -59,6 +56,9 @@ class TestDashboardMetadata(unittest.TestCase):
         self.expected_nodes = copy.deepcopy(self.expected_nodes_deduped)
 
         self.expected_rels_deduped = [
+            {'END_KEY': '_dashboard://gold.Product - Jobs.cz', 'END_LABEL': 'Dashboardgroup',
+             'REVERSE_TYPE': 'DASHBOARD_GROUP_OF', 'START_KEY': '_dashboard://gold',
+             'START_LABEL': 'Cluster', 'TYPE': 'DASHBOARD_GROUP'},
             {'END_KEY': '_dashboard://gold.Product - Jobs.cz/_description', 'END_LABEL': 'Description',
              'REVERSE_TYPE': 'DESCRIPTION_OF', 'START_KEY': '_dashboard://gold.Product - Jobs.cz',
              'START_LABEL': 'Dashboardgroup', 'TYPE': 'DESCRIPTION'},
@@ -70,33 +70,29 @@ class TestDashboardMetadata(unittest.TestCase):
              'END_LABEL': 'Description',
              'START_KEY': '_dashboard://gold.Product - Jobs.cz/Agent', 'TYPE': 'DESCRIPTION',
              'REVERSE_TYPE': 'DESCRIPTION_OF'},
-            {'END_KEY': '_dashboard://gold.Product - Jobs.cz/Agent/_lastreloadtime', 'START_LABEL': 'Dashboard',
-             'END_LABEL': 'Lastreloadtime', 'START_KEY': '_dashboard://gold.Product - Jobs.cz/Agent',
-             'TYPE': 'LAST_RELOAD_TIME', 'REVERSE_TYPE': 'LAST_RELOAD_TIME_OF'},
             {'END_KEY': 'test_tag', 'START_LABEL': 'Dashboard', 'END_LABEL': 'Tag',
              'START_KEY': '_dashboard://gold.Product - Jobs.cz/Agent', 'TYPE': 'TAG', 'REVERSE_TYPE': 'TAG_OF'},
             {'END_KEY': 'tag2', 'START_LABEL': 'Dashboard', 'END_LABEL': 'Tag',
-             'START_KEY': '_dashboard://gold.Product - Jobs.cz/Agent', 'TYPE': 'TAG', 'REVERSE_TYPE': 'TAG_OF'},
-            {'END_KEY': 'roald.amundsen@example.org', 'START_LABEL': 'Dashboard', 'END_LABEL': 'User',
-             'START_KEY': '_dashboard://gold.Product - Jobs.cz/Agent', 'TYPE': 'OWNER', 'REVERSE_TYPE': 'OWNER_OF'}
+             'START_KEY': '_dashboard://gold.Product - Jobs.cz/Agent', 'TYPE': 'TAG', 'REVERSE_TYPE': 'TAG_OF'}
         ]
 
         self.expected_rels = copy.deepcopy(self.expected_rels_deduped)
 
         self.expected_nodes_deduped2 = [
+            {'KEY': '_dashboard://gold', 'LABEL': 'Cluster', 'name': 'gold'},
             {'name': 'Atmoskop', 'KEY': '_dashboard://gold.Product - Atmoskop/Atmoskop', 'LABEL': 'Dashboard'},
             {'name': 'Product - Atmoskop', 'KEY': '_dashboard://gold.Product - Atmoskop', 'LABEL': 'Dashboardgroup'},
             {'description': 'Atmoskop dashboard description',
              'KEY': '_dashboard://gold.Product - Atmoskop/Atmoskop/_description',
              'LABEL': 'Description'},
-            {'value': '2019-05-30T07:07:42.326Z',
-             'KEY': '_dashboard://gold.Product - Atmoskop/Atmoskop/_lastreloadtime',
-             'LABEL': 'Lastreloadtime'}
         ]
 
         self.expected_nodes2 = copy.deepcopy(self.expected_nodes_deduped2)
 
         self.expected_rels_deduped2 = [
+            {'END_KEY': '_dashboard://gold.Product - Atmoskop', 'END_LABEL': 'Dashboardgroup',
+             'REVERSE_TYPE': 'DASHBOARD_GROUP_OF', 'START_KEY': '_dashboard://gold',
+             'START_LABEL': 'Cluster', 'TYPE': 'DASHBOARD_GROUP'},
             {'END_KEY': '_dashboard://gold.Product - Atmoskop', 'START_LABEL': 'Dashboard',
              'END_LABEL': 'Dashboardgroup',
              'START_KEY': '_dashboard://gold.Product - Atmoskop/Atmoskop', 'TYPE': 'DASHBOARD_OF',
@@ -105,20 +101,14 @@ class TestDashboardMetadata(unittest.TestCase):
              'END_LABEL': 'Description',
              'START_KEY': '_dashboard://gold.Product - Atmoskop/Atmoskop', 'TYPE': 'DESCRIPTION',
              'REVERSE_TYPE': 'DESCRIPTION_OF'},
-            {'END_KEY': '_dashboard://gold.Product - Atmoskop/Atmoskop/_lastreloadtime', 'START_LABEL': 'Dashboard',
-             'END_LABEL': 'Lastreloadtime', 'START_KEY': '_dashboard://gold.Product - Atmoskop/Atmoskop',
-             'TYPE': 'LAST_RELOAD_TIME', 'REVERSE_TYPE': 'LAST_RELOAD_TIME_OF'},
-            {'END_KEY': 'buzz@example.org', 'START_LABEL': 'Dashboard', 'END_LABEL': 'User',
-             'START_KEY': '_dashboard://gold.Product - Atmoskop/Atmoskop', 'TYPE': 'OWNER', 'REVERSE_TYPE': 'OWNER_OF'}
         ]
 
         self.expected_rels2 = copy.deepcopy(self.expected_rels_deduped2)
 
         self.expected_nodes_deduped3 = [
+            {'KEY': '_dashboard://gold', 'LABEL': 'Cluster', 'name': 'gold'},
             {'name': 'Dohazovac', 'KEY': '_dashboard://gold.Product - Jobs.cz/Dohazovac', 'LABEL': 'Dashboard'},
             {'name': 'Product - Jobs.cz', 'KEY': '_dashboard://gold.Product - Jobs.cz', 'LABEL': 'Dashboardgroup'},
-            {'value': '2019-05-30T07:07:42.326Z',
-             'KEY': '_dashboard://gold.Product - Jobs.cz/Dohazovac/_lastreloadtime', 'LABEL': 'Lastreloadtime'},
             {'tag_type': 'dashboard', 'KEY': 'test_tag', 'LABEL': 'Tag'},
             {'tag_type': 'dashboard', 'KEY': 'tag3', 'LABEL': 'Tag'}
         ]
@@ -126,19 +116,17 @@ class TestDashboardMetadata(unittest.TestCase):
         self.expected_nodes3 = copy.deepcopy(self.expected_nodes_deduped3)
 
         self.expected_rels_deduped3 = [
+            {'END_KEY': '_dashboard://gold.Product - Jobs.cz', 'END_LABEL': 'Dashboardgroup',
+             'REVERSE_TYPE': 'DASHBOARD_GROUP_OF', 'START_KEY': '_dashboard://gold',
+             'START_LABEL': 'Cluster', 'TYPE': 'DASHBOARD_GROUP'},
             {'END_KEY': '_dashboard://gold.Product - Jobs.cz', 'START_LABEL': 'Dashboard',
              'END_LABEL': 'Dashboardgroup',
              'START_KEY': '_dashboard://gold.Product - Jobs.cz/Dohazovac', 'TYPE': 'DASHBOARD_OF',
              'REVERSE_TYPE': 'DASHBOARD'},
-            {'END_KEY': '_dashboard://gold.Product - Jobs.cz/Dohazovac/_lastreloadtime', 'START_LABEL': 'Dashboard',
-             'END_LABEL': 'Lastreloadtime', 'START_KEY': '_dashboard://gold.Product - Jobs.cz/Dohazovac',
-             'TYPE': 'LAST_RELOAD_TIME', 'REVERSE_TYPE': 'LAST_RELOAD_TIME_OF'},
             {'END_KEY': 'test_tag', 'START_LABEL': 'Dashboard', 'END_LABEL': 'Tag',
              'START_KEY': '_dashboard://gold.Product - Jobs.cz/Dohazovac', 'TYPE': 'TAG', 'REVERSE_TYPE': 'TAG_OF'},
             {'END_KEY': 'tag3', 'START_LABEL': 'Dashboard', 'END_LABEL': 'Tag',
              'START_KEY': '_dashboard://gold.Product - Jobs.cz/Dohazovac', 'TYPE': 'TAG', 'REVERSE_TYPE': 'TAG_OF'},
-            {'END_KEY': 'buzz@example.org', 'START_LABEL': 'Dashboard', 'END_LABEL': 'User',
-             'START_KEY': '_dashboard://gold.Product - Jobs.cz/Dohazovac', 'TYPE': 'OWNER', 'REVERSE_TYPE': 'OWNER_OF'},
         ]
 
         self.expected_rels3 = copy.deepcopy(self.expected_rels_deduped3)

tests/unit/models/test_table_last_updated.py

@@ -4,6 +4,7 @@ from databuilder.models.neo4j_csv_serde import NODE_KEY, \
     NODE_LABEL, RELATION_START_KEY, RELATION_START_LABEL, RELATION_END_KEY, \
     RELATION_END_LABEL, RELATION_TYPE, RELATION_REVERSE_TYPE
 from databuilder.models.table_last_updated import TableLastUpdated
+from databuilder.models.timestamp import timestamp_constants
 
 
 class TestTableLastUpdated(unittest.TestCase):
@@ -20,6 +21,7 @@ class TestTableLastUpdated(unittest.TestCase):
             NODE_KEY: 'hive://gold.default/test_table/timestamp',
             NODE_LABEL: 'Timestamp',
             'last_updated_timestamp': 25195665,
+            timestamp_constants.TIMESTAMP_PROPERTY: 25195665,
             'name': 'last_updated_timestamp'
         }
 
@@ -35,12 +37,12 @@ class TestTableLastUpdated(unittest.TestCase):
     def test_create_next_node(self):
         # type: () -> None
         next_node = self.tableLastUpdated.create_next_node()
-        self.assertEquals(next_node, self.expected_node_result)
+        self.assertEqual(next_node, self.expected_node_result)
 
     def test_create_next_relation(self):
         # type: () -> None
         next_relation = self.tableLastUpdated.create_next_relation()
-        self.assertEquals(next_relation, self.expected_relation_result)
+        self.assertEqual(next_relation, self.expected_relation_result)
 
     def test_get_table_model_key(self):
         # type: () -> None

tests/unit/transformer/test_template_variable_substitution_transformer.py

+import unittest
+
+from pyhocon import ConfigFactory
+
+from databuilder.transformer.template_variable_substitution_transformer import \
+    TemplateVariableSubstitutionTransformer, FIELD_NAME, TEMPLATE
+
+
+class TestTemplateVariableSubstitutionTransformer(unittest.TestCase):
+
+    def test_conversion(self):
+        # type: () -> None
+
+        transformer = TemplateVariableSubstitutionTransformer()
+        config = ConfigFactory.from_dict({
+            FIELD_NAME: 'baz',
+            TEMPLATE: 'Hello {foo}'
+        })
+        transformer.init(conf=config)
+
+        actual = transformer.transform({'foo': 'bar'})
+        expected = {
+            'foo': 'bar',
+            'baz': 'Hello bar'
+        }
+        self.assertDictEqual(expected, actual)
+
+
+if __name__ == '__main__':
+    unittest.main()