Unverified Commit bfc6f810 authored by Dorian Johnson's avatar Dorian Johnson Committed by GitHub

build: fix broken tests in Python 3.7, test in CI (#334)

parent 86f658d1
language: python
python:
python:
- '3.6'
- '3.7'
install:
- pip install -r requirements.txt
- pip install .[all]
......
# Copyright Contributors to the Amundsen project.
# SPDX-License-Identifier: Apache-2.0
import itertools
import logging
import unittest
from datetime import datetime
......@@ -17,6 +18,15 @@ from databuilder.filesystem.filesystem import FileSystem
from databuilder.filesystem.metadata import FileMetadata
def null_iterator(items):
"""
Returns an infinite iterator that returns the items from items,
then infinite Nones. Required because Extractor.extract is expected
to return None when it is exhausted, not terminate.
"""
return itertools.chain(iter(items), itertools.repeat(None))
class TestHiveTableLastUpdatedExtractor(unittest.TestCase):
def setUp(self):
......@@ -52,14 +62,14 @@ class TestHiveTableLastUpdatedExtractor(unittest.TestCase):
return_value=pt_alchemy_extractor_instance),\
patch.object(HiveTableLastUpdatedExtractor, '_get_non_partitioned_table_sql_alchemy_extractor',
return_value=non_pt_alchemy_extractor_instance):
pt_alchemy_extractor_instance.extract = MagicMock(side_effect=[
pt_alchemy_extractor_instance.extract = MagicMock(side_effect=null_iterator([
{'schema': 'foo_schema',
'table_name': 'table_1',
'last_updated_time': 1},
{'schema': 'foo_schema',
'table_name': 'table_2',
'last_updated_time': 2}
])
]))
non_pt_alchemy_extractor_instance.extract = MagicMock(return_value=None)
......@@ -102,11 +112,11 @@ class TestHiveTableLastUpdatedExtractor(unittest.TestCase):
'_get_filesystem', return_value=fs):
pt_alchemy_extractor_instance.extract = MagicMock(return_value=None)
non_pt_alchemy_extractor_instance.extract = MagicMock(side_effect=[
non_pt_alchemy_extractor_instance.extract = MagicMock(side_effect=null_iterator([
{'schema': 'foo_schema',
'table_name': 'table_1',
'location': '/foo/bar'}
])
'location': '/foo/bar'},
]))
extractor = HiveTableLastUpdatedExtractor()
extractor.init(ConfigFactory.from_dict({}))
......
......@@ -30,11 +30,13 @@ class TestModePaginatedRestApiQuery(unittest.TestCase):
{'foo': [{'name': 'v3'}]},
{'foo': [{'name': 'v4'}, {'name': 'v5'}]},
{'foo': [{'name': 'v4'}, {'name': 'v5'}]},
{},
{}
]
query = ModePaginatedRestApiQuery(query_to_join=seed_query, url='foobar', params={},
json_path=json_path, field_names=field_names,
pagination_json_path='foo[*]',
skip_no_result=True, pagination_json_path='foo[*]',
max_record_size=2)
expected_list = [
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment