feat: Added tags and badges into Dashboard Elasticsearch index (#288)

* Added tags and badges into Dashboard Elasticsearch index

* Update

* Update

databuilder/extractor/neo4j_search_data_extractor.py

@@ -81,24 +81,32 @@ class Neo4jSearchDataExtractor(Extractor):
 
     DEFAULT_NEO4J_DASHBOARD_CYPHER_QUERY = textwrap.dedent(
         """
-        MATCH (db:Dashboard)
-        MATCH (db)-[:DASHBOARD_OF]->(dbg:Dashboardgroup)
+        MATCH (dashboard:Dashboard)
+        {publish_tag_filter}
+        MATCH (dashboard)-[:DASHBOARD_OF]->(dbg:Dashboardgroup)
         MATCH (dbg)-[:DASHBOARD_GROUP_OF]->(cluster:Cluster)
-        OPTIONAL MATCH (db)-[:DESCRIPTION]->(db_descr:Description)
+        OPTIONAL MATCH (dashboard)-[:DESCRIPTION]->(db_descr:Description)
         OPTIONAL MATCH (dbg)-[:DESCRIPTION]->(dbg_descr:Description)
-        OPTIONAL MATCH (db)-[:EXECUTED]->(last_exec:Execution)
+        OPTIONAL MATCH (dashboard)-[:EXECUTED]->(last_exec:Execution)
         WHERE split(last_exec.key, '/')[5] = '_last_successful_execution'
-        OPTIONAL MATCH (db)-[read:READ_BY]->(user:User)
-        OPTIONAL MATCH (db)-[:HAS_QUERY]->(query:Query)
-        with db, dbg, db_descr, dbg_descr, cluster, last_exec, query, SUM(read.read_count) AS total_usage
-        return dbg.name as group_name, db.name as name, cluster.name as cluster,
+        OPTIONAL MATCH (dashboard)-[read:READ_BY]->(user:User)
+        WITH dashboard, dbg, db_descr, dbg_descr, cluster, last_exec, SUM(read.read_count) AS total_usage
+        OPTIONAL MATCH (dashboard)-[:HAS_QUERY]->(query:Query)
+        WITH dashboard, dbg, db_descr, dbg_descr, cluster, last_exec, COLLECT(DISTINCT query.name) as query_names,
+        total_usage
+        OPTIONAL MATCH (dashboard)-[:TAGGED_BY]->(tags:Tag) WHERE tags.tag_type='default'
+        WITH dashboard, dbg, db_descr, dbg_descr, cluster, last_exec, query_names, total_usage,
+        COLLECT(DISTINCT tags.key) as tags
+        OPTIONAL MATCH (dashboard)-[:TAGGED_BY]->(badges:Tag) WHERE badges.tag_type='badge'
+        WITH  dashboard, dbg, db_descr, dbg_descr, cluster, last_exec, query_names, total_usage, tags,
+        COLLECT(DISTINCT badges.key) as badges
+        RETURN dbg.name as group_name, dashboard.name as name, cluster.name as cluster,
         coalesce(db_descr.description, '') as description,
         coalesce(dbg.description, '') as group_description, dbg.dashboard_group_url as group_url,
-        db.dashboard_url as url, db.key as uri,
-        split(db.key, '_')[0] as product, toInt(last_exec.timestamp) as last_successful_run_timestamp,
-        COLLECT(DISTINCT query.name) as query_names,
-        total_usage
-        order by dbg.name
+        dashboard.dashboard_url as url, dashboard.key as uri,
+        split(dashboard.key, '_')[0] as product, toInteger(last_exec.timestamp) as last_successful_run_timestamp,
+        query_names, total_usage, tags, badges
+        order by dbg.name;
         """
     )
 

databuilder/models/dashboard_elasticsearch_document.py

@@ -20,7 +20,8 @@ class DashboardESDocument(ElasticsearchDocument):
                  url=None,  # type: Optional[str]
                  uri=None,  # type: Optional[str]
                  last_successful_run_timestamp=None,  # type: Optional[int]
-                 tags=None  # type: list
+                 tags=None,  # type: Optional[list[str]]
+                 badges=None,  # type: Optional[list[str]]
                  ):
         # type: (...) -> None
         self.group_name = group_name
@@ -36,3 +37,4 @@ class DashboardESDocument(ElasticsearchDocument):
         self.group_description = group_description
         self.query_names = query_names
         self.tags = tags
+        self.badges = badges

databuilder/publisher/elasticsearch_constants.py

@@ -96,6 +96,17 @@ TABLE_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent(
 DASHBOARD_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent(
     """
     {
+        "settings": {
+          "analysis": {
+            "normalizer": {
+              "lowercase_normalizer": {
+                "type": "custom",
+                "char_filter": [],
+                "filter": ["lowercase", "asciifolding"]
+              }
+            }
+          }
+        },
         "mappings":{
             "dashboard":{
               "properties": {
@@ -104,7 +115,8 @@ DASHBOARD_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent(
                   "analyzer": "simple",
                   "fields": {
                     "raw": {
-                      "type": "keyword"
+                      "type": "keyword",
+                      "normalizer": "lowercase_normalizer"
                     }
                   }
                 },
@@ -113,7 +125,8 @@ DASHBOARD_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent(
                   "analyzer": "simple",
                   "fields": {
                     "raw": {
-                      "type": "keyword"
+                      "type": "keyword",
+                      "normalizer": "lowercase_normalizer"
                     }
                   }
                 },
@@ -143,12 +156,17 @@ DASHBOARD_ELASTICSEARCH_INDEX_MAPPING = textwrap.dedent(
                       "type": "keyword"
                     }
                   }
+                },
+                "tags": {
+                  "type": "keyword"
+                },
+                "badges": {
+                  "type": "keyword"
                 }
               }
             }
           }
         }
-
     """
 )
 

setup.py

@@ -2,7 +2,7 @@ import os
 from setuptools import setup, find_packages
 
 
-__version__ = '2.6.2'
+__version__ = '2.6.3'
 
 requirements_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'requirements.txt')
 with open(requirements_path) as requirements_file:

tests/unit/models/test_dashboard_elasticsearch_document.py

@@ -23,7 +23,8 @@ class TestDashboardElasticsearchDocument(unittest.TestCase):
                                        uri='mode_dashboard://gold.cluster/dashboard_group/dashboard',
                                        last_successful_run_timestamp=10,
                                        total_usage=10,
-                                       tags=['test'])
+                                       tags=['test'],
+                                       badges=['test_badge'])
 
         expected_document_dict = {"group_name": "test_dashboard_group",
                                   "name": "test_dashboard_name",
@@ -37,7 +38,9 @@ class TestDashboardElasticsearchDocument(unittest.TestCase):
                                   "last_successful_run_timestamp": 10,
                                   "group_description": "work space group",
                                   "total_usage": 10,
-                                  "tags": ["test"]
+                                  "tags": ["test"],
+                                  "badges": ["test_badge"],
+
                                   }
 
         result = test_obj.to_json()