Skip to content

A
amundsen_dev
Unverified Commit 29393825 authored by Tamika Tannis's avatar Tamika Tannis Committed by GitHub
Browse files
Options

Improve granularity of logging search actions (#396) 

* WIP: One approach

* Second approach

* Fix a few errors; Add tests

* Fix test

* Code cleanup

* Change value for when user selects inline result & searchAll is needed

* Update SearchType enum

* Use snake_case for consistency in backend until we have some auto-convert
parent 5aaec51b
Hide whitespace changes
Inline Side-by-side
Showing with 230 additions and 371 deletions
amundsen_application/api/search/v0.py
View file @ 29393825
......@@ -11,8 +11,7 @@ from flask.blueprints import Blueprint
from amundsen_application.log.action_log import action_logging
from amundsen_application.api.utils.request_utils import get_query_param, request_search
from amundsen_application.api.utils.response_utils import create_error_response
from amundsen_application.api.utils.search_utils import generate_query_json, map_table_result, valid_search_fields
from amundsen_application.api.utils.search_utils import generate_query_json, map_table_result
from amundsen_application.models.user import load_user, dump_user
LOGGER = logging.getLogger(__name__)
......@@ -21,151 +20,53 @@ REQUEST_SESSION_TIMEOUT_SEC = 3
search_blueprint = Blueprint('search', __name__, url_prefix='/api/search/v0')
SEARCH_ENDPOINT = '/search'
SEARCH_TABLE_ENDPOINT = '/search_table'
SEARCH_USER_ENDPOINT = '/search_user'
# TODO: To be deprecated pending full community support
def _validate_search_term(*, search_term: str, page_index: int) -> Optional[Response]:
error_payload = {
'results': [],
'search_term': search_term,
'total_results': 0,
'page_index': page_index,
}
# use colon means user would like to search on specific fields
if search_term.count(':') > 1:
message = 'Encountered error: Search field should not be more than 1'
return create_error_response(message=message, payload=error_payload, status_code=HTTPStatus.BAD_REQUEST)
if search_term.count(':') == 1:
field_key = search_term.split(' ')[0].split(':')[0]
if field_key not in valid_search_fields:
message = 'Encountered error: Search field is invalid'
return create_error_response(message=message, payload=error_payload, status_code=HTTPStatus.BAD_REQUEST)
return None
# TODO: To be deprecated pending full community support
@search_blueprint.route('/table', methods=['GET'])
@search_blueprint.route('/table', methods=['POST'])
def search_table() -> Response:
search_term = get_query_param(request.args, 'query', 'Endpoint takes a "query" parameter')
page_index = get_query_param(request.args, 'page_index', 'Endpoint takes a "page_index" parameter')
error_response = _validate_search_term(search_term=search_term, page_index=int(page_index))
if error_response is not None:
return error_response
results_dict = _search_table(search_term=search_term, page_index=page_index)
return make_response(jsonify(results_dict), results_dict.get('status_code', HTTPStatus.INTERNAL_SERVER_ERROR))
# TODO: To be deprecated pending full community support
@action_logging
def _search_table(*, search_term: str, page_index: int) -> Dict[str, Any]:
"""
call the search service endpoint and return matching results
:return: a json output containing search results array as 'results'
Schema Defined Here:
https://github.com/lyft/amundsensearchlibrary/blob/master/search_service/api/search.py
TODO: Define an interface for envoy_client
Parse the request arguments and call the helper method to execute a table search
:return: a Response created with the results from the helper method
"""
tables = {
'page_index': int(page_index),
'results': [],
'total_results': 0,
}
results_dict = {
'search_term': search_term,
'msg': '',
'tables': tables,
}
try:
if ':' in search_term:
url = _create_url_with_field(search_term=search_term,
page_index=page_index)
else:
url = '{0}?query_term={1}&page_index={2}'.format(app.config['SEARCHSERVICE_BASE'] + SEARCH_ENDPOINT,
search_term,
page_index)
request_json = request.get_json()
response = request_search(url=url)
status_code = response.status_code
search_term = get_query_param(request_json, 'term', '"term" parameter expected in request data')
page_index = get_query_param(request_json, 'pageIndex', '"pageIndex" parameter expected in request data')
if status_code == HTTPStatus.OK:
results_dict['msg'] = 'Success'
results = response.json().get('results')
tables['results'] = [map_table_result(result) for result in results]
tables['total_results'] = response.json().get('total_results')
else:
message = 'Encountered error: Search request failed'
results_dict['msg'] = message
logging.error(message)
search_type = request_json.get('searchType')
results_dict['status_code'] = status_code
return results_dict
filters = request_json.get('filters', {})
results_dict = _search_table(filters=filters,
search_term=search_term,
page_index=page_index,
search_type=search_type)
return make_response(jsonify(results_dict), results_dict.get('status_code', HTTPStatus.INTERNAL_SERVER_ERROR))
except Exception as e:
message = 'Encountered exception: ' + str(e)
results_dict['msg'] = message
logging.exception(message)
return results_dict
return make_response(jsonify(results_dict), HTTPStatus.INTERNAL_SERVER_ERROR)
# TODO: To be deprecated pending full community support
def _create_url_with_field(*, search_term: str, page_index: int) -> str:
"""
Construct a url by searching specific field.
E.g if we use search tag:hive test_table, search service will first
filter all the results that
don't have tag hive; then it uses test_table as query term to search /
rank all the documents.
We currently allow max 1 field.
todo: allow search multiple fields(e.g tag:hive & schema:default test_table)
:param search_term:
:param page_index:
:return:
"""
# example search_term: tag:tag_name search_term search_term2
fields = search_term.split(' ')
search_field = fields[0].split(':')
field_key = search_field[0]
# dedup tag to all lower case
field_val = search_field[1].lower()
search_term = ' '.join(fields[1:])
url = '{0}/field/{1}/field_val/{2}' \
'?page_index={3}'.format(app.config['SEARCHSERVICE_BASE'] + SEARCH_ENDPOINT,
field_key,
field_val,
page_index)
if search_term:
url += '&query_term={0}'.format(search_term)
return url
@search_blueprint.route('/table_qs', methods=['POST'])
def search_table_query_string() -> Response:
"""
TODO (ttannis): Update this docstring after amundsensearch documentation is merged
Calls the search service to execute a search. The request data is transformed
to the json payload defined [link]
@action_logging
def _search_table(*, search_term: str, page_index: int, filters: Dict, search_type: str) -> Dict[str, Any]:
"""
request_json = request.get_json()
search_term = get_query_param(request_json, 'term', '"term" parameter expected in request data')
page_index = get_query_param(request_json, 'pageIndex', '"pageIndex" parameter expected in request data')
filters = request_json.get('filters', {})
Call the search service endpoint and return matching results
Search service logic defined here:
https://github.com/lyft/amundsensearchlibrary/blob/master/search_service/api/table.py
:return: a json output containing search results array as 'results'
"""
# Default results
tables = {
'page_index': int(page_index),
'results': [],
'total_results': 0,
}
results_dict = {
'search_term': search_term,
'msg': '',
......@@ -178,11 +79,10 @@ def search_table_query_string() -> Response:
message = 'Encountered exception generating query json: ' + str(e)
results_dict['msg'] = message
logging.exception(message)
return make_response(jsonify(results_dict), HTTPStatus.INTERNAL_SERVER_ERROR)
return results_dict
try:
# TODO (ttannis): Change actual endpoint name after amundsensearch PR is merged
url = app.config['SEARCHSERVICE_BASE'] + '/search_table'
url = app.config['SEARCHSERVICE_BASE'] + SEARCH_TABLE_ENDPOINT
response = request_search(url=url,
headers={'Content-Type': 'application/json'},
method='POST',
......@@ -199,32 +99,42 @@ def search_table_query_string() -> Response:
logging.error(message)
results_dict['status_code'] = status_code
return make_response(jsonify(results_dict), status_code)
return results_dict
except Exception as e:
message = 'Encountered exception: ' + str(e)
results_dict['msg'] = message
logging.exception(message)
return make_response(jsonify(results_dict), HTTPStatus.INTERNAL_SERVER_ERROR)
return results_dict
@search_blueprint.route('/user', methods=['GET'])
def search_user() -> Response:
search_term = get_query_param(request.args, 'query', 'Endpoint takes a "query" parameter')
page_index = get_query_param(request.args, 'page_index', 'Endpoint takes a "page_index" parameter')
results_dict = _search_user(search_term=search_term, page_index=page_index)
"""
Parse the request arguments and call the helper method to execute a user search
:return: a Response created with the results from the helper method
"""
try:
search_term = get_query_param(request.args, 'query', 'Endpoint takes a "query" parameter')
page_index = get_query_param(request.args, 'page_index', 'Endpoint takes a "page_index" parameter')
search_type = request.args.get('search_type')
results_dict = _search_user(search_term=search_term, page_index=page_index, search_type=search_type)
return make_response(jsonify(results_dict), results_dict.get('status_code', HTTPStatus.INTERNAL_SERVER_ERROR))
return make_response(jsonify(results_dict), results_dict.get('status_code', HTTPStatus.INTERNAL_SERVER_ERROR))
except Exception as e:
message = 'Encountered exception: ' + str(e)
logging.exception(message)
return make_response(jsonify(results_dict), HTTPStatus.INTERNAL_SERVER_ERROR)
@action_logging
def _search_user(*, search_term: str, page_index: int) -> Dict[str, Any]:
def _search_user(*, search_term: str, page_index: int, search_type: str) -> Dict[str, Any]:
"""
call the search service endpoint and return matching results
:return: a json output containing search results array as 'results'
Schema Defined Here:
Call the search service endpoint and return matching results
Search service logic defined here:
https://github.com/lyft/amundsensearchlibrary/blob/master/search_service/api/user.py
TODO: Define an interface for envoy_client
:return: a json output containing search results array as 'results'
"""
def _map_user_result(result: Dict) -> Dict:
......@@ -268,10 +178,11 @@ def _search_user(*, search_term: str, page_index: int) -> Dict[str, Any]:
except Exception as e:
message = 'Encountered exception: ' + str(e)
results_dict['msg'] = message
results_dict['status_code'] = HTTPStatus.INTERNAL_SERVER_ERROR
logging.exception(message)
return results_dict
# TODO - Implement
def _search_dashboard(*, search_term: str, page_index: int) -> Dict[str, Any]:
def _search_dashboard(*, search_term: str, page_index: int, filters: Dict, search_type: str) -> Dict[str, Any]:
return {}
amundsen_application/api/utils/search_utils.py
View file @ 29393825
......@@ -23,11 +23,11 @@ def map_table_result(result: Dict) -> Dict:
}
def generate_query_json(*, filters: Dict = {}, page_index: str, search_term: str) -> Dict:
def generate_query_json(*, filters: Dict = {}, page_index: int, search_term: str) -> Dict:
"""
Transforms the given paramaters to the query json for the search service according to
the api defined at:
TODO (ttannis): Add link when amundsensearch PR is complete
https://github.com/lyft/amundsensearchlibrary/blob/master/search_service/api/swagger_doc/table/search_table_filter.yml
"""
# Generate the filter payload
filter_payload = {}
......
amundsen_application/static/js/ducks/search/api/tests/index.spec.ts
View file @ 29393825
......@@ -4,7 +4,7 @@ import { DashboardSearchResults, TableSearchResults, UserSearchResults } from 'd
import globalState from 'fixtures/globalState';
import { ResourceType } from 'interfaces';
import { ResourceType, SearchType } from 'interfaces';
import * as API from '../v0';
......@@ -44,7 +44,7 @@ describe('searchResource', () => {
const resourceType = ResourceType.dashboard;
const term = 'test';
expect.assertions(3);
await API.searchResource(pageIndex, resourceType, term).then(results => {
await API.searchResource(pageIndex, resourceType, term, undefined, SearchType.FILTER).then(results => {
expect(results).toEqual({});
});
expect(axiosMockGet).not.toHaveBeenCalled();
......@@ -59,7 +59,7 @@ describe('searchResource', () => {
const resourceType = ResourceType.user;
const term = 'test';
expect.assertions(3);
await API.searchResource(pageIndex, resourceType, term).then(results => {
await API.searchResource(pageIndex, resourceType, term, undefined, SearchType.FILTER).then(results => {
expect(results).toEqual({});
});
expect(axiosMockGet).not.toHaveBeenCalled();
......@@ -74,14 +74,15 @@ describe('searchResource', () => {
const pageIndex = 0;
const resourceType = ResourceType.user;
const term = 'test';
await API.searchResource(pageIndex, resourceType, term);
expect(axiosMockGet).toHaveBeenCalledWith(`${API.BASE_URL}/${resourceType}?query=${term}&page_index=${pageIndex}`);
const searchType = SearchType.SUBMIT_TERM;
await API.searchResource(pageIndex, resourceType, term, undefined, searchType);
expect(axiosMockGet).toHaveBeenCalledWith(`${API.BASE_URL}/${resourceType}?query=${term}&page_index=${pageIndex}&search_type=${searchType}`);
expect(axiosMockPost).not.toHaveBeenCalled();
});
it('calls searchResourceHelper with api call response', async () => {
const searchResourceHelperSpy = jest.spyOn(API, 'searchResourceHelper');
await API.searchResource(0, ResourceType.user, 'test');
await API.searchResource(0, ResourceType.user, 'test', undefined, SearchType.FILTER);
expect(searchResourceHelperSpy).toHaveBeenCalledWith(mockSearchResponse);
});
})
......@@ -94,18 +95,20 @@ describe('searchResource', () => {
const resourceType = ResourceType.table;
const term = 'test';
const filters = { 'schema': 'schema_name' }
await API.searchResource(pageIndex, resourceType, term, filters);
const searchType = SearchType.SUBMIT_TERM;
await API.searchResource(pageIndex, resourceType, term, filters, searchType);
expect(axiosMockGet).not.toHaveBeenCalled();
expect(axiosMockPost).toHaveBeenCalledWith(`${API.BASE_URL}/${resourceType}_qs`, {
expect(axiosMockPost).toHaveBeenCalledWith(`${API.BASE_URL}/${resourceType}`, {
filters,
pageIndex,
term,
searchType,
});
});
it('calls searchResourceHelper with api call response', async () => {
const searchResourceHelperSpy = jest.spyOn(API, 'searchResourceHelper');
await API.searchResource(0, ResourceType.table, 'test', { 'schema': 'schema_name' });
await API.searchResource(0, ResourceType.table, 'test', { 'schema': 'schema_name' }, SearchType.FILTER);
expect(searchResourceHelperSpy).toHaveBeenCalledWith(mockSearchResponse);
});
})
......
amundsen_application/static/js/ducks/search/api/v0.ts
View file @ 29393825
import axios, { AxiosResponse } from 'axios';
import { indexUsersEnabled } from 'config/config-utils';
import { ResourceType } from 'interfaces';
import { ResourceType, SearchType } from 'interfaces';
import { DashboardSearchResults, TableSearchResults, UserSearchResults } from '../types';
......@@ -29,7 +29,7 @@ export const searchResourceHelper = (response: AxiosResponse<SearchAPI>) => {
return ret;
};
export function searchResource(pageIndex: number, resource: ResourceType, term: string, filters: ResourceFilterReducerState = {}) {
export function searchResource(pageIndex: number, resource: ResourceType, term: string, filters: ResourceFilterReducerState = {}, searchType: SearchType) {
if (resource === ResourceType.dashboard ||
(resource === ResourceType.user && !indexUsersEnabled())) {
return Promise.resolve({});
......@@ -37,12 +37,13 @@ export function searchResource(pageIndex: number, resource: ResourceType, term:
/* Note: This logic must exist until query string endpoints are created for all resources */
if (resource === ResourceType.table) {
return axios.post(`${BASE_URL}/${resource}_qs`, {
return axios.post(`${BASE_URL}/${resource}`, {
filters,
pageIndex,
term,
searchType,
}).then(searchResourceHelper);
}
return axios.get(`${BASE_URL}/${resource}?query=${term}&page_index=${pageIndex}`)
return axios.get(`${BASE_URL}/${resource}?query=${term}&page_index=${pageIndex}&search_type=${searchType}`)
.then(searchResourceHelper);
};
amundsen_application/static/js/ducks/search/reducer.ts
View file @ 29393825
import { ResourceType } from 'interfaces';
import { ResourceType, SearchType} from 'interfaces';
import { Search as UrlSearch } from 'history';
......@@ -49,13 +49,14 @@ export interface SearchReducerState {
};
/* ACTIONS */
export function searchAll(term: string, resource?: ResourceType, pageIndex?: number, useFilters: boolean = false): SearchAllRequest {
export function searchAll(searchType: SearchType, term: string, resource?: ResourceType, pageIndex?: number, useFilters: boolean = false): SearchAllRequest {
return {
payload: {
resource,
pageIndex,
term,
useFilters,
searchType,
},
type: SearchAll.REQUEST,
};
......@@ -67,12 +68,13 @@ export function searchAllFailure(): SearchAllResponse {
return { type: SearchAll.FAILURE };
};
export function searchResource(term: string, resource: ResourceType, pageIndex: number): SearchResourceRequest {
export function searchResource(searchType: SearchType, term: string, resource: ResourceType, pageIndex: number): SearchResourceRequest {
return {
payload: {
pageIndex,
term,
resource,
searchType
},
type: SearchResource.REQUEST,
};
......
amundsen_application/static/js/ducks/search/sagas.ts
View file @ 29393825
......@@ -3,7 +3,7 @@ import { all, call, debounce, put, select, takeEvery, takeLatest } from 'redux-s
import * as _ from 'lodash';
import * as qs from 'simple-query-string';
import { ResourceType } from 'interfaces/Resources';
import { ResourceType, SearchType } from 'interfaces';
import * as API from './api/v0';
......@@ -68,7 +68,7 @@ export function* filterWorker(): SagaIterator {
const state = yield select(getSearchState);
const { search_term, selectedTab, filters } = state;
const pageIndex = getPageIndex(state)
yield put(searchResource(search_term, selectedTab, pageIndex));
yield put(searchResource(SearchType.FILTER, search_term, selectedTab, pageIndex));
updateSearchUrl({ filters, resource: selectedTab, term: search_term, index: pageIndex }, true);
};
......@@ -90,7 +90,7 @@ export function* filterWatcher2(): SagaIterator {
export function* filterWorker2(action: any): SagaIterator {
const state = yield select(getSearchState);
const { pageIndex = 0, resourceType, term = '' } = action.payload;
yield put(searchResource(term, resourceType, pageIndex));
yield put(searchResource(SearchType.FILTER, term, resourceType, pageIndex));
updateSearchUrl({ term, filters: state.filters, resource: resourceType, index: pageIndex }, false);
};
......@@ -98,8 +98,8 @@ export function* inlineSearchWorker(action: InlineSearchRequest): SagaIterator {
const { term } = action.payload;
try {
const [tableResponse, userResponse] = yield all([
call(API.searchResource, 0, ResourceType.table, term),
call(API.searchResource, 0, ResourceType.user, term),
call(API.searchResource, 0, ResourceType.table, term, {}, SearchType.INLINE_SEARCH),
call(API.searchResource, 0, ResourceType.user, term, {}, SearchType.INLINE_SEARCH),
]);
const inlineSearchResponse = {
tables: tableResponse.tables || initialInlineResultsState.tables,
......@@ -124,7 +124,7 @@ export function* selectInlineResultWorker(action): SagaIterator {
const state = yield select();
const { searchTerm, resourceType, updateUrl } = action.payload;
if (state.search.inlineResults.isLoading) {
yield put(searchAll(searchTerm, resourceType, 0))
yield put(searchAll(SearchType.INLINE_SELECT, searchTerm, resourceType, 0, false))
updateSearchUrl({ term: searchTerm, filters: state.search.filters });
}
else {
......@@ -144,67 +144,10 @@ export function* selectInlineResultsWatcher(): SagaIterator {
yield takeEvery(InlineSearch.SELECT, selectInlineResultWorker);
};
export function* searchAllWorker(action: SearchAllRequest): SagaIterator {
let { resource } = action.payload;
const { pageIndex, term, useFilters } = action.payload;
if (!useFilters) {
yield put(clearAllFilters())
}
const state = yield select(getSearchState);
const tableIndex = resource === ResourceType.table ? pageIndex : 0;
const userIndex = resource === ResourceType.user ? pageIndex : 0;
const dashboardIndex = resource === ResourceType.dashboard ? pageIndex : 0;
try {
const [tableResponse, userResponse, dashboardResponse] = yield all([
call(API.searchResource, tableIndex, ResourceType.table, term, state.filters[ResourceType.table]),
call(API.searchResource, userIndex, ResourceType.user, term, state.filters[ResourceType.user]),
call(API.searchResource, dashboardIndex, ResourceType.dashboard, term, state.filters[ResourceType.dashboard]),
]);
const searchAllResponse = {
search_term: term,
selectedTab: resource,
tables: tableResponse.tables || initialState.tables,
users: userResponse.users || initialState.users,
dashboards: dashboardResponse.dashboards || initialState.dashboards,
isLoading: false,
};
if (resource === undefined) {
resource = autoSelectResource(searchAllResponse);
searchAllResponse.selectedTab = resource;
}
const index = getPageIndex(searchAllResponse);
yield put(searchAllSuccess(searchAllResponse));
updateSearchUrl({ term, resource, index, filters: state.filters }, true);
} catch (e) {
yield put(searchAllFailure());
}
};
export function* searchAllWatcher(): SagaIterator {
yield takeEvery(SearchAll.REQUEST, searchAllWorker);
};
export function* searchResourceWorker(action: SearchResourceRequest): SagaIterator {
const { pageIndex, resource, term } = action.payload;
const state = yield select(getSearchState);
try {
const searchResults = yield call(API.searchResource, pageIndex, resource, term, state.filters[resource]);
yield put(searchResourceSuccess(searchResults));
} catch (e) {
yield put(searchResourceFailure());
}
};
export function* searchResourceWatcher(): SagaIterator {
yield takeEvery(SearchResource.REQUEST, searchResourceWorker);
};
export function* submitSearchWorker(action: SubmitSearchRequest): SagaIterator {
const state = yield select(getSearchState);
const { searchTerm, useFilters } = action.payload;
yield put(searchAll(searchTerm, undefined, undefined, useFilters));
yield put(searchAll(SearchType.SUBMIT_TERM, searchTerm, undefined, undefined, useFilters));
updateSearchUrl({ term: searchTerm, filters: state.filters });
};
export function* submitSearchWatcher(): SagaIterator {
......@@ -230,7 +173,7 @@ export function* setResourceWatcher(): SagaIterator {
export function* setPageIndexWorker(action: SetPageIndexRequest): SagaIterator {
const { pageIndex, updateUrl } = action.payload;
const state = yield select(getSearchState);
yield put(searchResource(state.search_term, state.selectedTab, pageIndex));
yield put(searchResource(SearchType.PAGINATION, state.search_term, state.selectedTab, pageIndex));
if (updateUrl) {
updateSearchUrl({
......@@ -249,7 +192,7 @@ export function* clearSearchWorker(action: ClearSearchRequest): SagaIterator {
/* If there was a previous search term, search each resource using filters */
const state = yield select(getSearchState);
if (!!state.search_term) {
yield put(searchAll('', undefined, undefined, true));
yield put(searchAll(SearchType.CLEAR_TERM, '', undefined, undefined, true));
}
};
export function* clearSearchWatcher(): SagaIterator {
......@@ -264,7 +207,7 @@ export function* urlDidUpdateWorker(action: UrlDidUpdateRequest): SagaIterator {
const state = yield select(getSearchState);
if (!!term && state.search_term !== term) {
yield put(searchAll(term, resource, parsedIndex));
yield put(searchAll(SearchType.LOAD_URL, term, resource, parsedIndex));
} else if (!!resource) {
if (resource !== state.selectedTab) {
yield put(setResource(resource, false))
......@@ -302,3 +245,66 @@ export function* loadPreviousSearchWorker(action: LoadPreviousSearchRequest): Sa
export function* loadPreviousSearchWatcher(): SagaIterator {
yield takeEvery(LoadPreviousSearch.REQUEST, loadPreviousSearchWorker);
};
//////////////////////////////////////////////////////////////////////////////
// API/END SAGAS
// These sagas directly trigger axios search requests.
// The actions that trigger them should only be fired by other sagas,
// and these sagas should be considered the "end" of any saga chain.
//////////////////////////////////////////////////////////////////////////////
export function* searchResourceWorker(action: SearchResourceRequest): SagaIterator {
const { pageIndex, resource, term, searchType } = action.payload;
const state = yield select(getSearchState);
try {
const searchResults = yield call(API.searchResource, pageIndex, resource, term, state.filters[resource], searchType);
yield put(searchResourceSuccess(searchResults));
} catch (e) {
yield put(searchResourceFailure());
}
};
export function* searchResourceWatcher(): SagaIterator {
yield takeEvery(SearchResource.REQUEST, searchResourceWorker);
};
export function* searchAllWorker(action: SearchAllRequest): SagaIterator {
let { resource } = action.payload;
const { pageIndex, term, useFilters, searchType } = action.payload;
if (!useFilters) {
yield put(clearAllFilters())
}
const state = yield select(getSearchState);
const tableIndex = resource === ResourceType.table ? pageIndex : 0;
const userIndex = resource === ResourceType.user ? pageIndex : 0;
const dashboardIndex = resource === ResourceType.dashboard ? pageIndex : 0;
try {
const [tableResponse, userResponse, dashboardResponse] = yield all([
call(API.searchResource, tableIndex, ResourceType.table, term, state.filters[ResourceType.table], searchType),
call(API.searchResource, userIndex, ResourceType.user, term, state.filters[ResourceType.user], searchType),
call(API.searchResource, dashboardIndex, ResourceType.dashboard, term, state.filters[ResourceType.dashboard], searchType),
]);
const searchAllResponse = {
search_term: term,
selectedTab: resource,
tables: tableResponse.tables || initialState.tables,
users: userResponse.users || initialState.users,
dashboards: dashboardResponse.dashboards || initialState.dashboards,
isLoading: false,
};
if (resource === undefined) {
resource = autoSelectResource(searchAllResponse);
searchAllResponse.selectedTab = resource;
}
const index = getPageIndex(searchAllResponse);
yield put(searchAllSuccess(searchAllResponse));
updateSearchUrl({ term, resource, index, filters: state.filters }, true);
} catch (e) {
yield put(searchAllFailure());
}
};
export function* searchAllWatcher(): SagaIterator {
yield takeEvery(SearchAll.REQUEST, searchAllWorker);
};
amundsen_application/static/js/ducks/search/tests/index.spec.ts
View file @ 29393825
import { testSaga } from 'redux-saga-test-plan';
import { debounce } from 'redux-saga/effects';
import { DEFAULT_RESOURCE_TYPE, ResourceType } from 'interfaces';
import { DEFAULT_RESOURCE_TYPE, ResourceType, SearchType } from 'interfaces';
import * as NavigationUtils from 'utils/navigationUtils';
import * as SearchUtils from 'ducks/search/utils';
......@@ -146,26 +146,30 @@ describe('search ducks', () => {
const term = 'test';
const resource = ResourceType.table;
const pageIndex = 0;
const action = searchAll(term, resource, pageIndex);
const searchType = SearchType.SUBMIT_TERM;
const action = searchAll(searchType, term, resource, pageIndex);
const { payload } = action;
expect(action.type).toBe(SearchAll.REQUEST);
expect(payload.resource).toBe(resource);
expect(payload.term).toBe(term);
expect(payload.pageIndex).toBe(pageIndex);
expect(payload.useFilters).toBe(false);
expect(payload.searchType).toBe(searchType);
});
it('searchAll - returns the action to search all resources with useFilters', () => {
const term = 'test';
const resource = ResourceType.table;
const pageIndex = 0;
const action = searchAll(term, resource, pageIndex, true);
const searchType = SearchType.SUBMIT_TERM;
const action = searchAll(searchType, term, resource, pageIndex, true);
const { payload } = action;
expect(action.type).toBe(SearchAll.REQUEST);
expect(payload.resource).toBe(resource);
expect(payload.term).toBe(term);
expect(payload.pageIndex).toBe(pageIndex);
expect(payload.useFilters).toBe(true);
expect(payload.searchType).toBe(searchType);
});
it('searchAllSuccess - returns the action to process the success', () => {
......@@ -184,12 +188,14 @@ describe('search ducks', () => {
const term = 'test';
const resource = ResourceType.table;
const pageIndex = 0;
const action = searchResource(term, resource, pageIndex);
const searchType = SearchType.SUBMIT_TERM;
const action = searchResource(searchType, term, resource, pageIndex);
const { payload } = action;
expect(action.type).toBe(SearchResource.REQUEST);
expect(payload.resource).toBe(resource);
expect(payload.term).toBe(term);
expect(payload.pageIndex).toBe(pageIndex);
expect(payload.searchType).toBe(searchType);
});
it('searchResourceSuccess - returns the action to process the success', () => {
......@@ -309,11 +315,9 @@ describe('search ducks', () => {
expect(reducer(testState, { type: 'INVALID.ACTION' })).toEqual(testState);
});
it('should handle SearchAll.REQUEST', () => {
it('should handle SearchAll.REQUEST', () => {
const term = 'testSearch';
const resource = ResourceType.table;
const pageIndex = 0;
expect(reducer(testState, searchAll(term, resource, pageIndex))).toEqual({
expect(reducer(testState, searchAll(SearchType.SUBMIT_TERM, term, ResourceType.table, 0))).toEqual({
...testState,
inlineResults: initialInlineResultsState,
search_term: term,
......@@ -346,7 +350,7 @@ describe('search ducks', () => {
});
it('should handle SearchResource.REQUEST', () => {
expect(reducer(testState, searchResource('test', ResourceType.table, 0))).toEqual({
expect(reducer(testState, searchResource(SearchType.SUBMIT_TERM, 'test', ResourceType.table, 0))).toEqual({
...initialState,
isLoading: true,
});
......@@ -498,7 +502,7 @@ describe('search ducks', () => {
updateSearchUrlSpy.mockClear();
saga = saga.next().select(SearchUtils.getSearchState).next(mockSearchState);
expect(getPageIndexSpy).toHaveBeenCalledWith(mockSearchState);
saga = saga.put(searchResource(mockSearchState.search_term, mockSearchState.selectedTab, mockIndex)).next();
saga = saga.put(searchResource(SearchType.FILTER, mockSearchState.search_term, mockSearchState.selectedTab, mockIndex)).next();
expect(updateSearchUrlSpy).toHaveBeenCalledWith({
filters: mockSearchState.filters,
resource: mockSearchState.selectedTab,
......@@ -530,7 +534,7 @@ describe('search ducks', () => {
*/
it('handles request error', () => {
testSaga(Sagas.searchAllWorker, searchAll('test', ResourceType.table, 0, true))
testSaga(Sagas.searchAllWorker, searchAll(SearchType.SUBMIT_TERM, 'test', ResourceType.table, 0, true))
.next().select(SearchUtils.getSearchState)
.next(globalState.search).throw(new Error()).put(searchAllFailure())
.next().isDone();
......@@ -551,15 +555,16 @@ describe('search ducks', () => {
const resource = ResourceType.table;
const term = 'test';
const mockSearchState = globalState.search;
testSaga(Sagas.searchResourceWorker, searchResource(term, resource, pageIndex))
const searchType = SearchType.PAGINATION;
testSaga(Sagas.searchResourceWorker, searchResource(searchType, term, resource, pageIndex))
.next().select(SearchUtils.getSearchState)
.next(mockSearchState).call(API.searchResource, pageIndex, resource, term, mockSearchState.filters[resource])
.next(mockSearchState).call(API.searchResource, pageIndex, resource, term, mockSearchState.filters[resource], searchType)
.next(expectedSearchResults).put(searchResourceSuccess(expectedSearchResults))
.next().isDone();
});
it('handles request error', () => {
testSaga(Sagas.searchResourceWorker, searchResource('test', ResourceType.table, 0))
testSaga(Sagas.searchResourceWorker, searchResource(SearchType.PAGINATION, 'test', ResourceType.table, 0))
.next().select(SearchUtils.getSearchState)
.next(globalState.search).throw(new Error()).put(searchResourceFailure())
.next().isDone();
......@@ -573,7 +578,7 @@ describe('search ducks', () => {
updateSearchUrlSpy.mockClear();
testSaga(Sagas.submitSearchWorker, submitSearch(term, true))
.next().select(SearchUtils.getSearchState)
.next(mockSearchState).put(searchAll(term, undefined, undefined, true))
.next(mockSearchState).put(searchAll(SearchType.SUBMIT_TERM, term, undefined, undefined, true))
.next().isDone();
expect(updateSearchUrlSpy).toHaveBeenCalledWith({ term, filters: mockSearchState.filters });
......@@ -632,7 +637,7 @@ describe('search ducks', () => {
testSaga(Sagas.setPageIndexWorker, setPageIndex(index, updateUrl))
.next().select(SearchUtils.getSearchState)
.next(searchState).put(searchResource(searchState.search_term, searchState.selectedTab, index))
.next(searchState).put(searchResource(SearchType.PAGINATION, searchState.search_term, searchState.selectedTab, index))
.next().isDone();
expect(updateSearchUrlSpy).toHaveBeenCalled();
});
......@@ -644,7 +649,7 @@ describe('search ducks', () => {
testSaga(Sagas.setPageIndexWorker, setPageIndex(index, updateUrl))
.next().select(SearchUtils.getSearchState)
.next(searchState).put(searchResource(searchState.search_term, searchState.selectedTab, index))
.next(searchState).put(searchResource(SearchType.PAGINATION, searchState.search_term, searchState.selectedTab, index))
.next().isDone();
expect(updateSearchUrlSpy).not.toHaveBeenCalled();
});
......@@ -679,7 +684,7 @@ describe('search ducks', () => {
it('Calls searchAll when search term changes', () => {
term = 'new search';
sagaTest(urlDidUpdate(`term=${term}&resource=${resource}&index=${index}`))
.put(searchAll(term, resource, index))
.put(searchAll(SearchType.LOAD_URL, term, resource, index))
.next().isDone();
});
......
amundsen_application/static/js/ducks/search/types.ts
View file @ 29393825
......@@ -4,6 +4,7 @@ import {
DashboardResource,
Resource,
ResourceType,
SearchType,
TableResource,
UserResource,
} from 'interfaces';
......@@ -53,6 +54,7 @@ export interface SearchAllRequest {
pageIndex: number;
term: string;
useFilters?: boolean;
searchType: SearchType;
};
type: SearchAll.REQUEST;
};
......@@ -75,6 +77,7 @@ export interface SearchResourceRequest {
pageIndex: number;
resource: ResourceType;
term: string;
searchType: SearchType;
};
type: SearchResource.REQUEST;
};
......
amundsen_application/static/js/interfaces/Enums.ts
View file @ 29393825
......@@ -7,3 +7,13 @@ export enum FilterType {
CHECKBOX_SELECT = 'checkboxFilter',
INPUT_SELECT = 'inputFilter'
}
export enum SearchType {
CLEAR_TERM = 'clear_search_term',
FILTER = 'update_filter',
INLINE_SEARCH = 'inline_search',
INLINE_SELECT = 'inline_select',
LOAD_URL = 'load_url',
PAGINATION = 'update_page',
SUBMIT_TERM = 'submit_search_term',
}
tests/unit/api/search/test_v0.py
View file @ 29393825
......@@ -6,7 +6,7 @@ from http import HTTPStatus
from unittest.mock import patch
from amundsen_application import create_app
from amundsen_application.api.search.v0 import _create_url_with_field, SEARCH_ENDPOINT, SEARCH_USER_ENDPOINT
from amundsen_application.api.search.v0 import SEARCH_TABLE_ENDPOINT, SEARCH_USER_ENDPOINT
local_app = create_app('amundsen_application.config.TestConfig', 'tests/templates')
......@@ -49,7 +49,8 @@ class SearchTableQueryString(unittest.TestCase):
def setUp(self) -> None:
self.mock_table_results = MOCK_TABLE_RESULTS
self.expected_parsed_table_results = MOCK_PARSED_TABLE_RESULTS
self.search_url = local_app.config['SEARCHSERVICE_BASE'] + '/search_table'
self.search_service_url = local_app.config['SEARCHSERVICE_BASE'] + SEARCH_TABLE_ENDPOINT
self.fe_flask_endpoint = '/api/search/v0/table'
def test_fail_if_term_is_none(self) -> None:
"""
......@@ -57,7 +58,7 @@ class SearchTableQueryString(unittest.TestCase):
:return:
"""
with local_app.test_client() as test:
response = test.post('/api/search/v0/table_qs', json={'pageIndex': 0})
response = test.post(self.fe_flask_endpoint, json={'pageIndex': 0})
self.assertEqual(response.status_code, HTTPStatus.INTERNAL_SERVER_ERROR)
def test_fail_if_page_index_is_none(self) -> None:
......@@ -66,9 +67,35 @@ class SearchTableQueryString(unittest.TestCase):
:return:
"""
with local_app.test_client() as test:
response = test.post('/api/search/v0/table_qs', json={'term': ''})
response = test.post(self.fe_flask_endpoint, json={'term': ''})
self.assertEqual(response.status_code, HTTPStatus.INTERNAL_SERVER_ERROR)
@responses.activate
@patch('amundsen_application.api.search.v0._search_table')
def test_calls_search_table_log_helper(self, search_table_mock) -> None:
"""
Test _search_table helper method is called with correct arguments for logging
from the request_json
:return:
"""
test_filters = {'schema': 'test_schema'}
test_term = 'hello'
test_index = 1
test_search_type = 'test'
responses.add(responses.POST, self.search_service_url, json=self.mock_table_results, status=HTTPStatus.OK)
with local_app.test_client() as test:
test.post(self.fe_flask_endpoint,
json={
'term': test_term,
'pageIndex': test_index,
'filters': test_filters,
'searchType': test_search_type})
search_table_mock.assert_called_with(filters=test_filters,
page_index=test_index,
search_term=test_term,
search_type=test_search_type)
@responses.activate
@patch('amundsen_application.api.search.v0.generate_query_json')
def test_calls_generate_query_json(self, mock_generate_query_json) -> None:
......@@ -80,10 +107,10 @@ class SearchTableQueryString(unittest.TestCase):
test_filters = {'schema': 'test_schema'}
test_term = 'hello'
test_index = 1
responses.add(responses.POST, self.search_url, json=self.mock_table_results, status=HTTPStatus.OK)
responses.add(responses.POST, self.search_service_url, json=self.mock_table_results, status=HTTPStatus.OK)
with local_app.test_client() as test:
test.post('/api/search/v0/table_qs',
test.post(self.fe_flask_endpoint,
json={'term': test_term, 'pageIndex': test_index, 'filters': test_filters})
mock_generate_query_json.assert_called_with(filters=test_filters,
page_index=test_index,
......@@ -102,7 +129,7 @@ class SearchTableQueryString(unittest.TestCase):
mock_generate_query_json.side_effect = Exception('Test exception')
with local_app.test_client() as test:
response = test.post('/api/search/v0/table_qs',
response = test.post(self.fe_flask_endpoint,
json={'term': test_term, 'pageIndex': test_index, 'filters': test_filters})
data = json.loads(response.data)
self.assertEqual(data.get('msg'), 'Encountered exception generating query json: Test exception')
......@@ -117,10 +144,10 @@ class SearchTableQueryString(unittest.TestCase):
test_filters = {'schema': 'test_schema'}
test_term = 'hello'
test_index = 1
responses.add(responses.POST, self.search_url, json=self.mock_table_results, status=HTTPStatus.OK)
responses.add(responses.POST, self.search_service_url, json=self.mock_table_results, status=HTTPStatus.OK)
with local_app.test_client() as test:
response = test.post('/api/search/v0/table_qs',
response = test.post(self.fe_flask_endpoint,
json={'term': test_term, 'pageIndex': test_index, 'filters': test_filters})
data = json.loads(response.data)
self.assertEqual(response.status_code, HTTPStatus.OK)
......@@ -138,20 +165,18 @@ class SearchTableQueryString(unittest.TestCase):
test_filters = {'schema': 'test_schema'}
test_term = 'hello'
test_index = 1
responses.add(responses.POST, self.search_url, json={}, status=HTTPStatus.BAD_REQUEST)
responses.add(responses.POST, self.search_service_url, json={}, status=HTTPStatus.BAD_REQUEST)
with local_app.test_client() as test:
response = test.post('/api/search/v0/table_qs',
response = test.post(self.fe_flask_endpoint,
json={'term': test_term, 'pageIndex': test_index, 'filters': test_filters})
data = json.loads(response.data)
self.assertEqual(response.status_code, HTTPStatus.BAD_REQUEST)
self.assertEqual(data.get('msg'), 'Encountered error: Search request failed')
class SearchTest(unittest.TestCase):
class SearchUserTest(unittest.TestCase):
def setUp(self) -> None:
self.mock_search_table_results = MOCK_TABLE_RESULTS
self.expected_parsed_search_table_results = MOCK_PARSED_TABLE_RESULTS
self.mock_search_user_results = {
'total_results': 1,
'results': [
......@@ -194,114 +219,7 @@ class SearchTest(unittest.TestCase):
'total_results': 1,
'results': 'Bad results to trigger exception'
}
# ----- Table Search Tests ---- #
def test_search_table_fail_if_no_query(self) -> None:
"""
Test request failure if 'query' is not provided in the query string
to the search endpoint
:return:
"""
with local_app.test_client() as test:
response = test.get('/api/search/v0/table', query_string=dict(page_index='0'))
self.assertEqual(response.status_code, HTTPStatus.INTERNAL_SERVER_ERROR)
def test_search_table_fail_if_no_page_index(self) -> None:
"""
Test request failure if 'page_index' is not provided in the query string
to the search endpoint
:return:
"""
with local_app.test_client() as test:
response = test.get('/api/search/v0/table', query_string=dict(query='test'))
self.assertEqual(response.status_code, HTTPStatus.INTERNAL_SERVER_ERROR)
@responses.activate
def test_search_table_success(self) -> None:
"""
Test request success
:return:
"""
responses.add(responses.GET, local_app.config['SEARCHSERVICE_BASE'] + SEARCH_ENDPOINT,
json=self.mock_search_table_results, status=HTTPStatus.OK)
with local_app.test_client() as test:
response = test.get('/api/search/v0/table', query_string=dict(query='test', page_index='0'))
data = json.loads(response.data)
self.assertEqual(response.status_code, HTTPStatus.OK)
tables = data.get('tables')
self.assertEqual(tables.get('total_results'), self.mock_search_table_results.get('total_results'))
self.assertCountEqual(tables.get('results'), self.expected_parsed_search_table_results)
@responses.activate
def test_search_table_fail_on_non_200_response(self) -> None:
"""
Test request failure if search endpoint returns non-200 http code
:return:
"""
responses.add(responses.GET, local_app.config['SEARCHSERVICE_BASE'] + SEARCH_ENDPOINT,
json=self.mock_search_table_results, status=HTTPStatus.INTERNAL_SERVER_ERROR)
with local_app.test_client() as test:
response = test.get('/api/search/v0/table', query_string=dict(query='test', page_index='0'))
self.assertEqual(response.status_code, HTTPStatus.INTERNAL_SERVER_ERROR)
@responses.activate
def test_search_table_fail_on_proccessing_bad_response(self) -> None:
"""
Test catching exception if there is an error processing the results
from the search endpoint
:return:
"""
responses.add(responses.GET, local_app.config['SEARCHSERVICE_BASE'] + SEARCH_ENDPOINT,
json=self.bad_search_results, status=HTTPStatus.OK)
with local_app.test_client() as test:
response = test.get('/api/search/v0/table', query_string=dict(query='test', page_index='0'))
self.assertEqual(response.status_code, HTTPStatus.INTERNAL_SERVER_ERROR)
@responses.activate
def test_search_table_with_field(self) -> None:
"""
Test search request if user search with colon
:return:
"""
responses.add(responses.GET, local_app.config['SEARCHSERVICE_BASE'] + SEARCH_ENDPOINT,
json={}, status=HTTPStatus.OK)
with local_app.test_client() as test:
response = test.get('/api/search/field/'
'tag_names/field_val/test', query_string=dict(query_term='test',
page_index='0'))
self.assertEqual(response.status_code, HTTPStatus.OK)
def test_create_url_with_field(self) -> None:
# test with invalid search term
with self.assertRaises(Exception):
invalid_search_term1 = 'tag:hive & schema:default test'
_create_url_with_field(search_term=invalid_search_term1,
page_index=1)
invalid_search_term2 = 'tag1:hive tag'
_create_url_with_field(search_term=invalid_search_term2,
page_index=1)
with local_app.app_context():
# test single tag with query term
search_term = 'tag:hive test_table'
expected = local_app.config['SEARCHSERVICE_BASE'] + \
'/search/field/tag/field_val/hive?page_index=1&query_term=test_table'
self.assertEqual(_create_url_with_field(search_term=search_term,
page_index=1), expected)
# test single tag without query term
search_term = 'tag:hive'
expected = local_app.config['SEARCHSERVICE_BASE'] + \
'/search/field/tag/field_val/hive?page_index=1'
self.assertEqual(_create_url_with_field(search_term=search_term,
page_index=1), expected)
self.fe_flask_endpoint = '/api/search/v0/user'
def test_search_user_fail_if_no_query(self) -> None:
"""
......@@ -310,7 +228,7 @@ class SearchTest(unittest.TestCase):
:return:
"""
with local_app.test_client() as test:
response = test.get('/api/search/v0/user', query_string=dict(page_index='0'))
response = test.get(self.fe_flask_endpoint, query_string=dict(page_index='0'))
self.assertEqual(response.status_code, HTTPStatus.INTERNAL_SERVER_ERROR)
def test_search_user_fail_if_no_page_index(self) -> None:
......@@ -320,7 +238,7 @@ class SearchTest(unittest.TestCase):
:return:
"""
with local_app.test_client() as test:
response = test.get('/api/search/v0/user', query_string=dict(query='test'))
response = test.get(self.fe_flask_endpoint, query_string=dict(query='test'))
self.assertEqual(response.status_code, HTTPStatus.INTERNAL_SERVER_ERROR)
@responses.activate
......@@ -333,12 +251,12 @@ class SearchTest(unittest.TestCase):
json=self.mock_search_user_results, status=HTTPStatus.OK)
with local_app.test_client() as test:
response = test.get('/api/search/v0/user', query_string=dict(query='test', page_index='0'))
response = test.get(self.fe_flask_endpoint, query_string=dict(query='test', page_index='0'))
data = json.loads(response.data)
self.assertEqual(response.status_code, HTTPStatus.OK)
users = data.get('users')
self.assertEqual(users.get('total_results'), self.mock_search_table_results.get('total_results'))
self.assertEqual(users.get('total_results'), self.mock_search_user_results.get('total_results'))
self.assertCountEqual(users.get('results'), self.expected_parsed_search_user_results)
@responses.activate
......@@ -348,8 +266,8 @@ class SearchTest(unittest.TestCase):
:return:
"""
responses.add(responses.GET, local_app.config['SEARCHSERVICE_BASE'] + SEARCH_USER_ENDPOINT,
json=self.mock_search_table_results, status=HTTPStatus.INTERNAL_SERVER_ERROR)
json=self.mock_search_user_results, status=HTTPStatus.INTERNAL_SERVER_ERROR)
with local_app.test_client() as test:
response = test.get('/api/search/v0/user', query_string=dict(query='test', page_index='0'))
response = test.get(self.fe_flask_endpoint, query_string=dict(query='test', page_index='0'))
self.assertEqual(response.status_code, HTTPStatus.INTERNAL_SERVER_ERROR)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment