Use ModePaginatedRestApiQuery for all extractor that calls list reports API (#254)

databuilder/extractor/dashboard/mode_analytics/mode_dashboard_charts_extractor.py

@@ -6,6 +6,7 @@ from typing import Any  # noqa: F401
 from databuilder import Scoped
 from databuilder.extractor.base_extractor import Extractor
 from databuilder.extractor.dashboard.mode_analytics.mode_dashboard_utils import ModeDashboardUtils
+from databuilder.rest_api.mode_analytics.mode_paginated_rest_api_query import ModePaginatedRestApiQuery
 from databuilder.rest_api.rest_api_query import RestApiQuery
 from databuilder.transformer.base_transformer import ChainedTransformer
 from databuilder.transformer.dict_to_model import DictToModel, MODEL_CLASS
@@ -79,7 +80,7 @@ class ModeDashboardChartsExtractor(Extractor):
         report_url_template = 'https://app.mode.com/api/{organization}/spaces/{dashboard_group_id}/reports'
         json_path = '(_embedded.reports[*].token)'
         field_names = ['dashboard_id']
-        reports_query = RestApiQuery(query_to_join=spaces_query, url=report_url_template, params=params,
+        reports_query = ModePaginatedRestApiQuery(query_to_join=spaces_query, url=report_url_template, params=params,
                                                   json_path=json_path, field_names=field_names, skip_no_result=True)
 
         queries_url_template = 'https://app.mode.com/api/{organization}/reports/{dashboard_id}/queries'

databuilder/extractor/dashboard/mode_analytics/mode_dashboard_executions_extractor.py

@@ -6,6 +6,7 @@ from typing import Any  # noqa: F401
 from databuilder import Scoped
 from databuilder.extractor.base_extractor import Extractor
 from databuilder.extractor.dashboard.mode_analytics.mode_dashboard_utils import ModeDashboardUtils
+from databuilder.rest_api.mode_analytics.mode_paginated_rest_api_query import ModePaginatedRestApiQuery
 from databuilder.rest_api.rest_api_query import RestApiQuery
 from databuilder.transformer.base_transformer import ChainedTransformer
 from databuilder.transformer.dict_to_model import DictToModel, MODEL_CLASS
@@ -77,8 +78,9 @@ class ModeDashboardExecutionsExtractor(Extractor):
         url = 'https://app.mode.com/api/{organization}/spaces/{dashboard_group_id}/reports'
         json_path = '(_embedded.reports[*].token) | (_embedded.reports[*]._links.last_run.href)'
         field_names = ['dashboard_id', 'last_run_resource_path']
-        last_run_resource_path_query = RestApiQuery(query_to_join=spaces_query, url=url, params=params,
-                                                    json_path=json_path, field_names=field_names, skip_no_result=True,
+        last_run_resource_path_query = ModePaginatedRestApiQuery(query_to_join=spaces_query, url=url, params=params,
+                                                                 json_path=json_path, field_names=field_names,
+                                                                 skip_no_result=True,
                                                                  json_path_contains_or=True)
 
         # https://mode.com/developer/api-reference/analytics/report-runs/#getReportRun

databuilder/extractor/dashboard/mode_analytics/mode_dashboard_extractor.py

@@ -108,6 +108,5 @@ class ModeDashboardExtractor(Extractor):
         json_path = '_embedded.reports[*].[token,name,description,created_at]'
         field_names = ['dashboard_id', 'dashboard_name', 'description', 'created_timestamp']
         reports_query = ModePaginatedRestApiQuery(query_to_join=spaces_query, url=reports_url_template, params=params,
-                                                  json_path=json_path, field_names=field_names, skip_no_result=True,
-                                                  pagination_json_path='_embedded.reports[*]')
+                                                  json_path=json_path, field_names=field_names, skip_no_result=True)
         return reports_query

databuilder/extractor/dashboard/mode_analytics/mode_dashboard_last_modified_timestamp_extractor.py

@@ -6,7 +6,8 @@ from databuilder.extractor.dashboard.mode_analytics.mode_dashboard_executions_ex
     ModeDashboardExecutionsExtractor
 from databuilder.extractor.dashboard.mode_analytics.mode_dashboard_utils import ModeDashboardUtils
 from databuilder.extractor.restapi.rest_api_extractor import STATIC_RECORD_DICT
-from databuilder.rest_api.rest_api_query import RestApiQuery
+from databuilder.rest_api.mode_analytics.mode_paginated_rest_api_query import ModePaginatedRestApiQuery
+from databuilder.rest_api.rest_api_query import RestApiQuery  # noqa: F401
 from databuilder.transformer.dict_to_model import DictToModel, MODEL_CLASS
 from databuilder.transformer.timestamp_string_to_epoch import TimestampStringToEpoch, FIELD_NAME
 
@@ -56,7 +57,8 @@ class ModeDashboardLastModifiedTimestampExtractor(ModeDashboardExecutionsExtract
         url = 'https://app.mode.com/api/{organization}/spaces/{dashboard_group_id}/reports'
         json_path = '_embedded.reports[*].[token,edited_at]'
         field_names = ['dashboard_id', 'last_modified_timestamp']
-        last_modified_query = RestApiQuery(query_to_join=spaces_query, url=url, params=params,
-                                           json_path=json_path, field_names=field_names, skip_no_result=True)
+        last_modified_query = ModePaginatedRestApiQuery(query_to_join=spaces_query, url=url, params=params,
+                                                        json_path=json_path, field_names=field_names,
+                                                        skip_no_result=True)
 
         return last_modified_query

databuilder/extractor/dashboard/mode_analytics/mode_dashboard_last_successful_executions_extractor.py

@@ -7,7 +7,8 @@ from databuilder.extractor.dashboard.mode_analytics.mode_dashboard_executions_ex
 from databuilder.extractor.dashboard.mode_analytics.mode_dashboard_utils import ModeDashboardUtils
 from databuilder.extractor.restapi.rest_api_extractor import STATIC_RECORD_DICT
 from databuilder.models.dashboard.dashboard_execution import DashboardExecution
-from databuilder.rest_api.rest_api_query import RestApiQuery
+from databuilder.rest_api.mode_analytics.mode_paginated_rest_api_query import ModePaginatedRestApiQuery
+from databuilder.rest_api.rest_api_query import RestApiQuery  # noqa: F401
 
 LOGGER = logging.getLogger(__name__)
 
@@ -53,7 +54,8 @@ class ModeDashboardLastSuccessfulExecutionExtractor(ModeDashboardExecutionsExtra
         url = 'https://app.mode.com/api/{organization}/spaces/{dashboard_group_id}/reports'
         json_path = '_embedded.reports[*].[token,last_successfully_run_at]'
         field_names = ['dashboard_id', 'execution_timestamp']
-        last_successful_run_query = RestApiQuery(query_to_join=spaces_query, url=url, params=params,
-                                                 json_path=json_path, field_names=field_names, skip_no_result=True)
+        last_successful_run_query = ModePaginatedRestApiQuery(query_to_join=spaces_query, url=url, params=params,
+                                                              json_path=json_path, field_names=field_names,
+                                                              skip_no_result=True)
 
         return last_successful_run_query

databuilder/extractor/dashboard/mode_analytics/mode_dashboard_owner_extractor.py

@@ -6,6 +6,7 @@ from typing import Any  # noqa: F401
 from databuilder.extractor.base_extractor import Extractor
 from databuilder.extractor.dashboard.mode_analytics.mode_dashboard_utils import ModeDashboardUtils
 from databuilder.extractor.restapi.rest_api_extractor import MODEL_CLASS
+from databuilder.rest_api.mode_analytics.mode_paginated_rest_api_query import ModePaginatedRestApiQuery
 from databuilder.rest_api.rest_api_failure_handlers import HttpFailureSkipOnStatus
 from databuilder.rest_api.rest_api_query import RestApiQuery
 
@@ -61,8 +62,10 @@ class ModeDashboardOwnerExtractor(Extractor):
         # Reports
         json_path = '(_embedded.reports[*].token) | (_embedded.reports[*]._links.creator.href)'
         field_names = ['dashboard_id', 'creator_resource_path']
-        creator_resource_path_query = RestApiQuery(query_to_join=spaces_query, url=report_url_template, params=params,
-                                                   json_path=json_path, field_names=field_names, skip_no_result=True,
+        creator_resource_path_query = ModePaginatedRestApiQuery(query_to_join=spaces_query, url=report_url_template,
+                                                                params=params,
+                                                                json_path=json_path, field_names=field_names,
+                                                                skip_no_result=True,
                                                                 json_path_contains_or=True)
 
         json_path = 'email'

databuilder/extractor/dashboard/mode_analytics/mode_dashboard_queries_extractor.py

@@ -6,6 +6,7 @@ from typing import Any  # noqa: F401
 from databuilder import Scoped
 from databuilder.extractor.base_extractor import Extractor
 from databuilder.extractor.dashboard.mode_analytics.mode_dashboard_utils import ModeDashboardUtils
+from databuilder.rest_api.mode_analytics.mode_paginated_rest_api_query import ModePaginatedRestApiQuery
 from databuilder.rest_api.rest_api_query import RestApiQuery
 from databuilder.transformer.base_transformer import ChainedTransformer
 from databuilder.transformer.dict_to_model import DictToModel, MODEL_CLASS
@@ -81,7 +82,7 @@ class ModeDashboardQueriesExtractor(Extractor):
         url = 'https://app.mode.com/api/{organization}/spaces/{dashboard_group_id}/reports'
         json_path = '(_embedded.reports[*].token)'
         field_names = ['dashboard_id']
-        reports_query = RestApiQuery(query_to_join=spaces_query, url=url, params=params,
+        reports_query = ModePaginatedRestApiQuery(query_to_join=spaces_query, url=url, params=params,
                                                   json_path=json_path, field_names=field_names, skip_no_result=True)
 
         queries_url_template = 'https://app.mode.com/api/{organization}/reports/{dashboard_id}/queries'

databuilder/extractor/dashboard/mode_analytics/mode_dashboard_usage_extractor.py

@@ -5,8 +5,8 @@ from typing import Any  # noqa: F401
 
 from databuilder.extractor.base_extractor import Extractor
 from databuilder.extractor.dashboard.mode_analytics.mode_dashboard_utils import ModeDashboardUtils
-from databuilder.rest_api.rest_api_query import RestApiQuery
-
+from databuilder.rest_api.mode_analytics.mode_paginated_rest_api_query import ModePaginatedRestApiQuery
+from databuilder.rest_api.rest_api_query import RestApiQuery  # noqa: F401
 
 LOGGER = logging.getLogger(__name__)
 
@@ -54,6 +54,6 @@ class ModeDashboardUsageExtractor(Extractor):
         # and view_count
         json_path = '_embedded.reports[*].[token,view_count]'
         field_names = ['dashboard_id', 'accumulated_view_count']
-        reports_query = RestApiQuery(query_to_join=spaces_query, url=reports_url_template, params=params,
+        reports_query = ModePaginatedRestApiQuery(query_to_join=spaces_query, url=reports_url_template, params=params,
                                                   json_path=json_path, field_names=field_names, skip_no_result=True)
         return reports_query

databuilder/rest_api/mode_analytics/mode_paginated_rest_api_query.py

@@ -9,6 +9,7 @@ from databuilder.rest_api.rest_api_query import RestApiQuery
 #  How many records considers as full and indicating there might be next page? In list reports on space API, it's 30.
 DEFAULT_MAX_RECORD_SIZE = 30
 PAGE_SUFFIX_TEMPLATE = '?page={}'
+LIST_REPORTS_PAGINATION_JSON_PATH = '_embedded.reports[*]'  # So far this is the only paginated API that we need.
 
 LOGGER = logging.getLogger(__name__)
 
@@ -22,7 +23,7 @@ class ModePaginatedRestApiQuery(RestApiQuery):
     """
 
     def __init__(self,
-                 pagination_json_path,  # type: str
+                 pagination_json_path=LIST_REPORTS_PAGINATION_JSON_PATH,  # type: str
                  max_record_size=DEFAULT_MAX_RECORD_SIZE,  # type: int
                  **kwargs  # type: Any
                  ):