Commit 8592c668 authored by Saulius Grigaliunas's avatar Saulius Grigaliunas Committed by Tao Feng

Fix throttle handling in BigQueryTableUsageExtractor (#182)

`DELAY_TIME` was an undefined constant previously. This patch makes
delay timeout configurable using a config param.

Default of 100 seconds as API request quota limits are guaranteed to be
refilled after said time interval accoring to the documentation at:
https://cloud.google.com/compute/docs/api-rate-limits
parent 28e1b55b
......@@ -24,6 +24,7 @@ class BigQueryTableUsageExtractor(BaseBigQueryExtractor):
TIMESTAMP_KEY = 'timestamp'
_DEFAULT_SCOPES = ('https://www.googleapis.com/auth/cloud-platform',)
EMAIL_PATTERN = 'email_pattern'
DELAY_TIME = 'delay_time'
def init(self, conf):
# type: (ConfigTree) -> None
......@@ -33,6 +34,7 @@ class BigQueryTableUsageExtractor(BaseBigQueryExtractor):
(date.today() - timedelta(days=1)).strftime('%Y-%m-%dT00:00:00Z'))
self.email_pattern = conf.get_string(BigQueryTableUsageExtractor.EMAIL_PATTERN, None)
self.delay_time = conf.get_int(BigQueryTableUsageExtractor.DELAY_TIME, 100)
self.table_usage_counts = {}
self._count_usage()
......@@ -136,7 +138,7 @@ class BigQueryTableUsageExtractor(BaseBigQueryExtractor):
response = None
except Exception:
# Add a delay when BQ quota exceeds limitation
sleep(BigQueryTableUsageExtractor.DELAY_TIME)
sleep(self.delay_time)
def get_scope(self):
# type: () -> str
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment