fix: Set Tableau URLs (base + API) via config (#349)

Signed-off-by: Tao Feng <fengtao04@gmail.com>

databuilder/extractor/dashboard/tableau/tableau_dashboard_constants.py

 API_VERSION = 'api_version'
-TABLEAU_HOST = 'tableau_host'
+API_BASE_URL = 'api_base_url'
+TABLEAU_BASE_URL = 'tableau_base_url'
 SITE_NAME = 'site_name'
 TABLEAU_ACCESS_TOKEN_NAME = 'tableau_personal_access_token_name'
 TABLEAU_ACCESS_TOKEN_SECRET = 'tableau_personal_access_token_secret'

databuilder/extractor/dashboard/tableau/tableau_dashboard_extractor.py

@@ -26,7 +26,7 @@ class TableauGraphQLApiMetadataExtractor(TableauGraphQLApiExtractor):
 
     CLUSTER = const.CLUSTER
     EXCLUDED_PROJECTS = const.EXCLUDED_PROJECTS
-    TABLEAU_HOST = const.TABLEAU_HOST
+    TABLEAU_BASE_URL = const.TABLEAU_BASE_URL
 
     def execute(self) -> Iterator[Dict[str, Any]]:
         response = self.execute_query()
@@ -41,12 +41,12 @@ class TableauGraphQLApiMetadataExtractor(TableauGraphQLApiExtractor):
                 'dashboard_name': TableauDashboardUtils.sanitize_workbook_name(workbook['name']),
                 'description': workbook.get('description', ''),
                 'created_timestamp': workbook['createdAt'],
-                'dashboard_group_url': 'https://{}/#/projects/{}'.format(
-                    self._conf.get(TableauGraphQLApiMetadataExtractor.TABLEAU_HOST),
+                'dashboard_group_url': '{}/#/projects/{}'.format(
+                    self._conf.get(TableauGraphQLApiMetadataExtractor.TABLEAU_BASE_URL),
                     workbook['projectVizportalUrlId']
                 ),
-                'dashboard_url': 'https://{}/#/workbooks/{}/views'.format(
-                    self._conf.get(TableauGraphQLApiMetadataExtractor.TABLEAU_HOST),
+                'dashboard_url': '{}/#/workbooks/{}/views'.format(
+                    self._conf.get(TableauGraphQLApiMetadataExtractor.TABLEAU_BASE_URL),
                     workbook['vizportalUrlId']
                 ),
                 'cluster': self._conf.get_string(TableauGraphQLApiMetadataExtractor.CLUSTER)
@@ -66,11 +66,12 @@ class TableauDashboardExtractor(Extractor):
     Uses the Metadata API: https://help.tableau.com/current/api/metadata_api/en-us/index.html
     """
 
+    API_BASE_URL = const.API_BASE_URL
     API_VERSION = const.API_VERSION
     CLUSTER = const.CLUSTER
     EXCLUDED_PROJECTS = const.EXCLUDED_PROJECTS
     SITE_NAME = const.SITE_NAME
-    TABLEAU_HOST = const.TABLEAU_HOST
+    TABLEAU_BASE_URL = const.TABLEAU_BASE_URL
     TABLEAU_ACCESS_TOKEN_NAME = const.TABLEAU_ACCESS_TOKEN_NAME
     TABLEAU_ACCESS_TOKEN_SECRET = const.TABLEAU_ACCESS_TOKEN_SECRET
     VERIFY_REQUEST = const.VERIFY_REQUEST

databuilder/extractor/dashboard/tableau/tableau_dashboard_last_modified_extractor.py

@@ -52,11 +52,11 @@ class TableauDashboardLastModifiedExtractor(Extractor):
         Dashboard last modified timestamp (Workbook last modified timestamp)
     """
 
+    API_BASE_URL = const.API_BASE_URL
     API_VERSION = const.API_VERSION
     CLUSTER = const.CLUSTER
     EXCLUDED_PROJECTS = const.EXCLUDED_PROJECTS
     SITE_NAME = const.SITE_NAME
-    TABLEAU_HOST = const.TABLEAU_HOST
     TABLEAU_ACCESS_TOKEN_NAME = const.TABLEAU_ACCESS_TOKEN_NAME
     TABLEAU_ACCESS_TOKEN_SECRET = const.TABLEAU_ACCESS_TOKEN_SECRET
     VERIFY_REQUEST = const.VERIFY_REQUEST

databuilder/extractor/dashboard/tableau/tableau_dashboard_query_extractor.py

@@ -52,11 +52,11 @@ class TableauDashboardQueryExtractor(Extractor):
     workbook that uses the query.
     """
 
+    API_BASE_URL = const.API_BASE_URL
     API_VERSION = const.API_VERSION
     CLUSTER = const.CLUSTER
     EXCLUDED_PROJECTS = const.EXCLUDED_PROJECTS
     SITE_NAME = const.SITE_NAME
-    TABLEAU_HOST = const.TABLEAU_HOST
     TABLEAU_ACCESS_TOKEN_NAME = const.TABLEAU_ACCESS_TOKEN_NAME
     TABLEAU_ACCESS_TOKEN_SECRET = const.TABLEAU_ACCESS_TOKEN_SECRET
     VERIFY_REQUEST = const.VERIFY_REQUEST

databuilder/extractor/dashboard/tableau/tableau_dashboard_table_extractor.py

@@ -89,13 +89,13 @@ class TableauDashboardTableExtractor(Extractor):
     Assumes that all the nodes for both the dashboards and the tables have already been created.
     """
 
+    API_BASE_URL = const.API_BASE_URL
     API_VERSION = const.API_VERSION
     CLUSTER = const.CLUSTER
     DATABASE = const.DATABASE
     EXCLUDED_PROJECTS = const.EXCLUDED_PROJECTS
     EXTERNAL_CLUSTER_NAME = const.EXTERNAL_CLUSTER_NAME
     SITE_NAME = const.SITE_NAME
-    TABLEAU_HOST = const.TABLEAU_HOST
     TABLEAU_ACCESS_TOKEN_NAME = const.TABLEAU_ACCESS_TOKEN_NAME
     TABLEAU_ACCESS_TOKEN_SECRET = const.TABLEAU_ACCESS_TOKEN_SECRET
     VERIFY_REQUEST = const.VERIFY_REQUEST

databuilder/extractor/dashboard/tableau/tableau_dashboard_utils.py

@@ -65,9 +65,9 @@ class TableauGraphQLApiExtractor(Extractor):
     Base class for querying the Tableau Metdata API, which uses a GraphQL schema.
     """
 
+    API_BASE_URL = const.API_BASE_URL
     QUERY = 'query'
     QUERY_VARIABLES = 'query_variables'
-    TABLEAU_HOST = const.TABLEAU_HOST
     VERIFY_REQUEST = 'verify_request'
 
     def init(self, conf: ConfigTree) -> None:
@@ -76,11 +76,11 @@ class TableauGraphQLApiExtractor(Extractor):
         self._query = self._conf.get(TableauGraphQLApiExtractor.QUERY)
         self._iterator: Optional[Iterator[Dict[str, Any]]] = None
         self._static_dict = conf.get(STATIC_RECORD_DICT, dict())
-        self._metadata_url = 'https://{TABLEAU_HOST}/api/metadata/graphql'.format(
-            TABLEAU_HOST=self._conf.get_string(TableauGraphQLApiExtractor.TABLEAU_HOST)
+        self._metadata_url = '{api_base_url}/api/metadata/graphql'.format(
+            api_base_url=self._conf.get_string(TableauGraphQLApiExtractor.API_BASE_URL)
         )
         self._query_variables = self._conf.get(TableauGraphQLApiExtractor.QUERY_VARIABLES, {})
-        self._verify_request = self._conf.get(TableauGraphQLApiExtractor.VERIFY_REQUEST, True)
+        self._verify_request = self._conf.get(TableauGraphQLApiExtractor.VERIFY_REQUEST, None)
 
     def execute_query(self) -> Dict[str, Any]:
         """
@@ -95,12 +95,12 @@ class TableauGraphQLApiExtractor(Extractor):
             'X-Tableau-Auth': self._auth_token
         }
         params = {
-            'data': query_payload,
-            'headers': headers,
-            'verify': self._verify_request
+            'headers': headers
         }
+        if self._verify_request is not None:
+            params['verify'] = self._verify_request
 
-        response = requests.post(url=self._metadata_url, **params)
+        response = requests.post(url=self._metadata_url, data=query_payload, **params)
         return response.json()['data']
 
     def execute(self) -> Iterator[Dict[str, Any]]:
@@ -136,9 +136,9 @@ class TableauDashboardAuth:
     https://help.tableau.com/current/api/rest_api/en-us/REST/rest_api_concepts_auth.htm
     """
 
+    API_BASE_URL = const.API_BASE_URL
     API_VERSION = const.API_VERSION
     SITE_NAME = const.SITE_NAME
-    TABLEAU_HOST = const.TABLEAU_HOST
     TABLEAU_ACCESS_TOKEN_NAME = const.TABLEAU_ACCESS_TOKEN_NAME
     TABLEAU_ACCESS_TOKEN_SECRET = const.TABLEAU_ACCESS_TOKEN_SECRET
     VERIFY_REQUEST = const.VERIFY_REQUEST
@@ -150,8 +150,8 @@ class TableauDashboardAuth:
         self._access_token_secret = self._conf.get_string(TableauDashboardAuth.TABLEAU_ACCESS_TOKEN_SECRET)
         self._api_version = self._conf.get_string(TableauDashboardAuth.API_VERSION)
         self._site_name = self._conf.get_string(TableauDashboardAuth.SITE_NAME)
-        self._tableau_host = self._conf.get_string(TableauDashboardAuth.TABLEAU_HOST)
-        self._verify_request = self._conf.get(TableauDashboardAuth.VERIFY_REQUEST, True)
+        self._api_base_url = self._conf.get_string(TableauDashboardAuth.API_BASE_URL)
+        self._verify_request = self._conf.get(TableauDashboardAuth.VERIFY_REQUEST, None)
 
     @property
     def token(self) -> Optional[str]:
@@ -166,10 +166,11 @@ class TableauDashboardAuth:
         See https://help.tableau.com/current/api/rest_api/en-us/REST/rest_api_concepts_versions.htm
         for details or ask your Tableau server administrator.
         """
-        self._auth_url = "https://{tableau_host}/api/{api_version}/auth/signin".format(
-            tableau_host=self._tableau_host,
+        self._auth_url = "{api_base_url}/api/{api_version}/auth/signin".format(
+            api_base_url=self._api_base_url,
             api_version=self._api_version
         )
+
         payload = json.dumps({
             'credentials': {
                 'personalAccessTokenName': self._access_token_name,
@@ -185,9 +186,10 @@ class TableauDashboardAuth:
         }
         # verify = False is needed bypass occasional (valid) self-signed cert errors. TODO: actually fix it!!
         params = {
-            'headers': headers,
-            'verify': self._verify_request
+            'headers': headers
         }
+        if self._verify_request is not None:
+            params['verify'] = self._verify_request
 
         response_json = requests.post(url=self._auth_url, data=payload, **params).json()
         return response_json['credentials']['token']

databuilder/extractor/dashboard/tableau/tableau_external_table_extractor.py

@@ -76,6 +76,7 @@ class TableauDashboardExternalTableExtractor(Extractor):
     googlesheets://external.growth_by_region_county/FY_20_Report
     """
 
+    API_BASE_URL = const.API_BASE_URL
     API_VERSION = const.API_VERSION
     CLUSTER = const.CLUSTER
     EXCLUDED_PROJECTS = const.EXCLUDED_PROJECTS
@@ -83,7 +84,6 @@ class TableauDashboardExternalTableExtractor(Extractor):
     EXTERNAL_SCHEMA_NAME = const.EXTERNAL_SCHEMA_NAME
     EXTERNAL_TABLE_TYPES = const.EXTERNAL_TABLE_TYPES
     SITE_NAME = const.SITE_NAME
-    TABLEAU_HOST = const.TABLEAU_HOST
     TABLEAU_ACCESS_TOKEN_NAME = const.TABLEAU_ACCESS_TOKEN_NAME
     TABLEAU_ACCESS_TOKEN_SECRET = const.TABLEAU_ACCESS_TOKEN_SECRET
     VERIFY_REQUEST = const.VERIFY_REQUEST

example/scripts/sample_tableau_data_loader.py

@@ -68,7 +68,8 @@ neo4j_password = 'test'
 
 LOGGER = logging.getLogger(__name__)
 
-tableau_host = ""
+tableau_base_url = ""
+tableau_api_base_url = ""
 tableau_api_version = 0
 tableau_site_name = ""
 tableau_personal_access_token_name = ""
@@ -79,7 +80,7 @@ tableau_dashboard_database = ""
 tableau_external_table_cluster = ""
 tableau_external_table_schema = ""
 tableau_external_table_types = []
-tableau_verify_request = True
+tableau_verify_request = None
 
 common_tableau_config = {
     'publisher.neo4j.neo4j_endpoint': neo4j_endpoint,
@@ -156,7 +157,8 @@ def run_tableau_metadata_job():
 
     dict_config = common_tableau_config
     dict_config.update({
-        'extractor.tableau_dashboard_metadata.tableau_host': tableau_host,
+        'extractor.tableau_dashboard_metadata.api_base_url': tableau_api_base_url,
+        'extractor.tableau_dashboard_metadata.tableau_base_url': tableau_base_url,
         'extractor.tableau_dashboard_metadata.api_version': tableau_api_version,
         'extractor.tableau_dashboard_metadata.site_name': tableau_site_name,
         'extractor.tableau_dashboard_metadata.tableau_personal_access_token_name':
@@ -195,7 +197,7 @@ def run_tableau_last_modified_job():
 
     dict_config = common_tableau_config
     dict_config.update({
-        'extractor.tableau_dashboard_last_modified.tableau_host': tableau_host,
+        'extractor.tableau_dashboard_last_modified.api_base_url': tableau_api_base_url,
         'extractor.tableau_dashboard_last_modified.api_version': tableau_api_version,
         'extractor.tableau_dashboard_last_modified.site_name': tableau_site_name,
         'extractor.tableau_dashboard_last_modified.tableau_personal_access_token_name':
@@ -234,7 +236,7 @@ def run_tableau_query_job():
 
     dict_config = common_tableau_config
     dict_config.update({
-        'extractor.tableau_dashboard_query.tableau_host': tableau_host,
+        'extractor.tableau_dashboard_query.api_base_url': tableau_api_base_url,
         'extractor.tableau_dashboard_query.api_version': tableau_api_version,
         'extractor.tableau_dashboard_query.site_name': tableau_site_name,
         'extractor.tableau_dashboard_query.tableau_personal_access_token_name':
@@ -273,7 +275,7 @@ def run_tableau_table_job():
 
     dict_config = common_tableau_config
     dict_config.update({
-        'extractor.tableau_dashboard_table.tableau_host': tableau_host,
+        'extractor.tableau_dashboard_table.api_base_url': tableau_api_base_url,
         'extractor.tableau_dashboard_table.api_version': tableau_api_version,
         'extractor.tableau_dashboard_table.site_name': tableau_site_name,
         'extractor.tableau_dashboard_table.tableau_personal_access_token_name':
@@ -313,7 +315,7 @@ def run_tableau_external_table_job():
 
     dict_config = common_tableau_config
     dict_config.update({
-        'extractor.tableau_external_table.tableau_host': tableau_host,
+        'extractor.tableau_external_table.api_base_url': tableau_api_base_url,
         'extractor.tableau_external_table.api_version': tableau_api_version,
         'extractor.tableau_external_table.site_name': tableau_site_name,
         'extractor.tableau_external_table.tableau_personal_access_token_name':

tests/unit/extractor/dashboard/tableau/test_tableau_dashboard_extractor.py

@@ -44,7 +44,8 @@ class TestTableauDashboardExtractor(unittest.TestCase):
     def test_dashboard_metadata_extractor(self) -> None:
 
         config = ConfigFactory.from_dict({
-            'extractor.tableau_dashboard_metadata.tableau_host': 'tableau_host',
+            'extractor.tableau_dashboard_metadata.api_base_url': 'api_base_url',
+            'extractor.tableau_dashboard_metadata.tableau_base_url': 'tableau_base_url',
             'extractor.tableau_dashboard_metadata.api_version': 'tableau_api_version',
             'extractor.tableau_dashboard_metadata.site_name': 'tableau_site_name',
             'extractor.tableau_dashboard_metadata.tableau_personal_access_token_name':

tests/unit/extractor/dashboard/tableau/test_tableau_dashboard_last_modified_extractor.py

@@ -42,7 +42,7 @@ class TestTableauDashboardLastModified(unittest.TestCase):
     def test_dashboard_last_modified_extractor(self) -> None:
 
         config = ConfigFactory.from_dict({
-            'extractor.tableau_dashboard_last_modified.tableau_host': 'tableau_host',
+            'extractor.tableau_dashboard_last_modified.api_base_url': 'api_base_url',
             'extractor.tableau_dashboard_last_modified.api_version': 'tableau_api_version',
             'extractor.tableau_dashboard_last_modified.site_name': 'tableau_site_name',
             'extractor.tableau_dashboard_last_modified.tableau_personal_access_token_name':

tests/unit/extractor/dashboard/tableau/test_tableau_dashboard_query_extractor.py

@@ -46,7 +46,7 @@ class TestTableauDashboardQuery(unittest.TestCase):
     def test_dashboard_query_extractor(self) -> None:
 
         config = ConfigFactory.from_dict({
-            'extractor.tableau_dashboard_query.tableau_host': 'tableau_host',
+            'extractor.tableau_dashboard_query.api_base_url': 'api_base_url',
             'extractor.tableau_dashboard_query.api_version': 'tableau_api_version',
             'extractor.tableau_dashboard_query.site_name': 'tableau_site_name',
             'extractor.tableau_dashboard_query.tableau_personal_access_token_name':

tests/unit/extractor/dashboard/tableau/test_tableau_dashboard_table_extractor.py

@@ -57,7 +57,7 @@ class TestTableauDashboardTable(unittest.TestCase):
     def test_dashboard_table_extractor(self) -> None:
 
         config = ConfigFactory.from_dict({
-            'extractor.tableau_dashboard_table.tableau_host': 'tableau_host',
+            'extractor.tableau_dashboard_table.api_base_url': 'api_base_url',
             'extractor.tableau_dashboard_table.api_version': 'tableau_api_version',
             'extractor.tableau_dashboard_table.site_name': 'tableau_site_name',
             'extractor.tableau_dashboard_table.tableau_personal_access_token_name':