use schemas for metadata retrieval & update search field names (#382)

* use schemas for metadata retrieval

* - Re-add user object sanitization
- Add 'first_name' and 'last_name' fallback for 'full_name'

* upgrade common to 0.2.2
Co-authored-by: Daniel <dwon@lyft.com>

amundsen_application/api/metadata/v0.py

@@ -121,7 +121,7 @@ def _get_table_metadata(*, table_key: str, index: int, source: str) -> Dict[str,
         return results_dict
 
     try:
-        table_data_raw = response.json()
+        table_data_raw: dict = response.json()
 
         # Ideally the response should include 'key' to begin with
         table_data_raw['key'] = table_key

amundsen_application/api/search/v0.py

@@ -186,8 +186,8 @@ def _search_table(*, search_term: str, page_index: int) -> Dict[str, Any]:
             'cluster': result.get('cluster', None),
             'description': result.get('description', None),
             'database': result.get('database', None),
-            'schema_name': result.get('schema_name', None),
-            'last_updated_epoch': result.get('last_updated_epoch', None),
+            'schema': result.get('schema', None),
+            'last_updated_timestamp': result.get('last_updated_timestamp', None),
         }
 
     tables = {

amundsen_application/api/utils/metadata_utils.py

-from typing import Dict
-from flask import current_app as app
+from typing import Any, Dict
 
+from amundsen_common.models.popular_table import PopularTable, PopularTableSchema
+from amundsen_common.models.table import Table, TableSchema
 from amundsen_application.models.user import load_user, dump_user
+from flask import current_app as app
 
 
-def marshall_table_partial(table: Dict) -> Dict:
+def marshall_table_partial(table_dict: Dict) -> Dict:
     """
     Forms a short version of a table Dict, with selected fields and an added 'key'
     :param table: Dict of partial table object
@@ -12,61 +14,41 @@ def marshall_table_partial(table: Dict) -> Dict:
 
     TODO - Unify data format returned by search and metadata.
     """
-    table_name = table.get('table_name', '')
-    schema_name = table.get('schema', '')
-    cluster = table.get('cluster', '')
-    db = table.get('database', '')
-    return {
-        'cluster': cluster,
-        'database': db,
-        'description': table.get('table_description', ''),
-        'key': '{0}://{1}.{2}/{3}'.format(db, cluster, schema_name, table_name),
-        'name': table_name,
-        'schema_name': schema_name,
-        'type': 'table',
-        'last_updated_epoch': table.get('last_updated_epoch', None),
-    }
+    schema = PopularTableSchema(strict=True)
+    # TODO: consider migrating to validate() instead of roundtripping
+    table: PopularTable = schema.load(table_dict).data
+    results = schema.dump(table).data
+    # TODO: fix popular tables to provide these? remove if we're not using them?
+    # TODO: Add the 'key' or 'id' to the base PopularTableSchema
+    results['key'] = f'{table.database}://{table.cluster}.{table.schema}/{ table.name}'
+    results['last_updated_timestamp'] = None
+    results['type'] = 'table'
+
+    return results
 
 
-def marshall_table_full(table: Dict) -> Dict:
+def marshall_table_full(table_dict: Dict) -> Dict:
     """
     Forms the full version of a table Dict, with additional and sanitized fields
     :param table: Table Dict from metadata service
     :return: Table Dict with sanitized fields
     """
-    # Filter and parse the response dictionary from the metadata service
-    fields = [
-        'badges',
-        'columns',
-        'cluster',
-        'database',
-        'is_view',
-        'key',
-        'owners',
-        'schema',
-        'source',
-        'table_description',
-        'table_name',
-        'table_readers',
-        'table_writer',
-        'tags',
-        'watermarks',
-        # 'last_updated_timestamp' Exists on the response from metadata but is not used.
-        # This should also be consolidated with 'last_updated_epoch' to have the same name and format.
-    ]
-
-    results = {field: table.get(field, None) for field in fields}
+
+    schema = TableSchema(strict=True)
+    # TODO: consider migrating to validate() instead of roundtripping
+    table: Table = schema.load(table_dict).data
+    results: Dict[str, Any] = schema.dump(table).data
 
     is_editable = results['schema'] not in app.config['UNEDITABLE_SCHEMAS']
     results['is_editable'] = is_editable
 
-    # In the list of owners, sanitize each entry
+    # TODO - Cleanup https://github.com/lyft/amundsen/issues/296
+    #  This code will try to supplement some missing data since the data here is incomplete.
+    #  Once the metadata service response provides complete user objects we can remove this.
     results['owners'] = [_map_user_object_to_schema(owner) for owner in results['owners']]
-
-    # In the list of reader_objects, sanitize the reader value on each entry
     readers = results['table_readers']
     for reader_object in readers:
-        reader_object['reader'] = _map_user_object_to_schema(reader_object['reader'])
+        reader_object['user'] = _map_user_object_to_schema(reader_object['user'])
 
     # If order is provided, we sort the column based on the pre-defined order
     if app.config['COLUMN_STAT_ORDER']:
@@ -77,6 +59,8 @@ def marshall_table_full(table: Dict) -> Dict:
                               get(x['stat_type'], len(app.config['COLUMN_STAT_ORDER'])))
             col['is_editable'] = is_editable
 
+    # TODO: Add the 'key' or 'id' to the base TableSchema
+    results['key'] = f'{table.database}://{table.cluster}.{table.schema}/{ table.name}'
     # Temp code to make 'partition_key' and 'partition_value' part of the table
     results['partition'] = _get_partition_data(results['watermarks'])
     return results

amundsen_application/models/user.py

@@ -8,6 +8,10 @@ from flask import current_app as app
 """
 TODO: Explore all internationalization use cases and
 redesign how User handles names
+
+TODO - Delete this file
+Once all of the upstream services provide a complete User object we will no
+longer need to supplement the User objects as done in `preprocess_data`
 """
 
 
@@ -81,8 +85,11 @@ class UserSchema(Schema):
             if app.config['GET_PROFILE_URL']:
                 data['profile_url'] = app.config['GET_PROFILE_URL'](data['user_id'])
 
-        # Fallback since search and metadata use a different key for 'full_name'
-        data['full_name'] = data.get('full_name', data.get('name'))
+        first_name = data.get('first_name')
+        last_name = data.get('last_name')
+
+        if self._str_no_value(data.get('full_name')) and first_name and last_name:
+            data['full_name'] = f"{first_name} {last_name}"
 
         if self. _str_no_value(data.get('display_name')):
             if self._str_no_value(data.get('full_name')):

amundsen_application/static/js/components/TableDetail/ColumnListItem/index.tsx

@@ -43,7 +43,7 @@ export class ColumnListItem extends React.Component<ColumnListItemProps, ColumnL
       logClick(e, {
         target_id: `column::${metadata.name}`,
         target_type: 'column stats',
-        label: `${metadata.name} ${metadata.type}`,
+        label: `${metadata.name} ${metadata.col_type}`,
       });
     }
     this.setState({ isExpanded: !this.state.isExpanded });
@@ -122,7 +122,7 @@ export class ColumnListItem extends React.Component<ColumnListItemProps, ColumnL
               }
             </div>
             <div className="resource-type">
-              { this.renderColumnType(this.props.index, metadata.type) }
+              { this.renderColumnType(this.props.index, metadata.col_type) }
             </div>
             <div className="badges">
               {/* Placeholder */}

amundsen_application/static/js/components/TableDetail/ColumnListItem/tests/index.spec.tsx

@@ -18,7 +18,7 @@ describe('ColumnListItem', () => {
         name: "test_column_name",
         description: "This is a test description of this table",
         is_editable: true,
-        type: "varchar(32)",
+        col_type: "varchar(32)",
         stats: [{ end_epoch: 1571616000, start_epoch: 1571616000, stat_type: "count", stat_val: "12345" }]
       },
       index: 0,
@@ -86,7 +86,7 @@ describe('ColumnListItem', () => {
 
     it('renders the correct resource type', () => {
       const resourceType = wrapper.find('.resource-type');
-      expect(resourceType.text()).toBe(props.data.type.toLowerCase());
+      expect(resourceType.text()).toBe(props.data.col_type.toLowerCase());
     });
 
     it('renders the dropdown when notifications is enabled', () => {

amundsen_application/static/js/components/TableDetail/DataPreviewButton/index.tsx

@@ -77,7 +77,7 @@ export class DataPreviewButton extends React.Component<DataPreviewButtonProps, D
     this.props.getPreviewData({
       database: tableData.database,
       schema: tableData.schema,
-      tableName: tableData.table_name,
+      tableName: tableData.name,
     });
   }
 

amundsen_application/static/js/components/TableDetail/ExploreButton/index.tsx

@@ -19,10 +19,10 @@ export class ExploreButton extends React.Component<ExploreButtonProps> {
 
     if (partition.is_partitioned) {
       return AppConfig.tableProfile.exploreUrlGenerator(
-        tableData.database, tableData.cluster, tableData.schema, tableData.table_name, partition.key, partition.value);
+        tableData.database, tableData.cluster, tableData.schema, tableData.name, partition.key, partition.value);
     }
     return AppConfig.tableProfile.exploreUrlGenerator(
-      tableData.database, tableData.cluster, tableData.schema, tableData.table_name);
+      tableData.database, tableData.cluster, tableData.schema, tableData.name);
   }
 
   render() {

amundsen_application/static/js/components/TableDetail/FrequentUsers/index.tsx

@@ -12,7 +12,7 @@ export interface FrequentUsersProps {
 }
 
 export function renderReader(reader: TableReader, index: number, readers: TableReader[]) {
-  const user = reader.reader;
+  const user = reader.user;
   let link = user.profile_url;
   let target = '_blank';
   if (AppConfig.indexUsers.enabled) {

amundsen_application/static/js/components/TableDetail/LineageLink/index.tsx

@@ -13,8 +13,8 @@ const LineageLink: React.SFC<LineageLinkProps> = ({ tableData }) => {
   const config = AppConfig.tableLineage;
   if (!config.isEnabled) return null;
 
-  const { database, cluster, schema, table_name } = tableData;
-  const href = config.urlGenerator(database, cluster, schema, table_name);
+  const { database, cluster, schema, name } = tableData;
+  const href = config.urlGenerator(database, cluster, schema, name);
   const label = 'Lineage';
 
   return (

amundsen_application/static/js/components/TableDetail/RequestMetadataForm/index.tsx

@@ -102,15 +102,15 @@ export class RequestMetadataForm extends React.Component<RequestMetadataProps, R
     const descriptionRequested = formData.get('table-description') === "on";
     const fieldsRequested = formData.get('column-description') === "on";
     const comment = formData.get('comment') as string;
-    const { cluster, database, schema, table_name } = this.props.tableMetadata;
+    const { cluster, database, schema, name } = this.props.tableMetadata;
     this.props.submitNotification(
       recipients,
       sender,
       NotificationType.METADATA_REQUESTED,
       {
         comment,
-        resource_name: `${schema}.${table_name}`,
-        resource_path: `/table_detail/${cluster}/${database}/${schema}/${table_name}`,
+        resource_name: `${schema}.${name}`,
+        resource_path: `/table_detail/${cluster}/${database}/${schema}/${name}`,
         description_requested: descriptionRequested,
         fields_requested: fieldsRequested,
       }

amundsen_application/static/js/components/TableDetail/RequestMetadataForm/tests/index.spec.tsx

@@ -114,7 +114,7 @@ describe('RequestMetadataForm', () => {
     it('calls submitNotification', () => {
       const { props, wrapper } = setup();
       const submitNotificationSpy = jest.spyOn(props, 'submitNotification');
-       const { cluster, database, schema, table_name } = props.tableMetadata;
+       const { cluster, database, schema, name } = props.tableMetadata;
       wrapper.instance().submitNotification({ preventDefault: jest.fn() });
       expect(submitNotificationSpy).toHaveBeenCalledWith(
         mockFormData['recipients'].split(','),
@@ -122,8 +122,8 @@ describe('RequestMetadataForm', () => {
         NotificationType.METADATA_REQUESTED,
         {
           comment: mockFormData['comment'],
-          resource_name: `${schema}.${table_name}`,
-          resource_path: `/table_detail/${cluster}/${database}/${schema}/${table_name}`,
+          resource_name: `${schema}.${name}`,
+          resource_path: `/table_detail/${cluster}/${database}/${schema}/${name}`,
           description_requested: true,
           fields_requested: false,
         }

amundsen_application/static/js/components/TableDetail/TableDescEditableText/index.tsx

@@ -8,7 +8,7 @@ import EditableText, { ComponentProps, DispatchFromProps, StateFromProps } from
 
 export const mapStateToProps = (state: GlobalState) => {
   return {
-    refreshValue: state.tableMetadata.tableData.table_description,
+    refreshValue: state.tableMetadata.tableData.description,
   };
 };
 

amundsen_application/static/js/components/TableDetail/index.tsx

@@ -170,7 +170,7 @@ class TableDetail extends React.Component<TableDetailProps & RouteComponentProps
               <EditableSection title="Description">
                 <TableDescEditableText
                   maxLength={ AppConfig.editableText.tableDescLength }
-                  value={ data.table_description }
+                  value={ data.description }
                   editable={ data.is_editable }
                 />
               </EditableSection>

amundsen_application/static/js/components/common/Bookmark/MyBookmarks/tests/index.spec.tsx

@@ -26,7 +26,7 @@ describe('MyBookmarks', () => {
           database: 'database',
           description: 'description',
           name: 'name',
-          schema_name: 'schema_name',
+          schema: 'schema',
         },
         {
           key: 'bookmark-2',
@@ -35,7 +35,7 @@ describe('MyBookmarks', () => {
           database: 'database',
           description: 'description',
           name: 'name',
-          schema_name: 'schema_name',
+          schema: 'schema',
         },
         {
           key: 'bookmark-3',
@@ -44,7 +44,7 @@ describe('MyBookmarks', () => {
           database: 'database',
           description: 'description',
           name: 'name',
-          schema_name: 'schema_name',
+          schema: 'schema',
         },
         {
           key: 'bookmark-4',
@@ -53,7 +53,7 @@ describe('MyBookmarks', () => {
           database: 'database',
           description: 'description',
           name: 'name',
-          schema_name: 'schema_name',
+          schema: 'schema',
         },
         {
           key: 'bookmark-5',
@@ -62,7 +62,7 @@ describe('MyBookmarks', () => {
           database: 'database',
           description: 'description',
           name: 'name',
-          schema_name: 'schema_name',
+          schema: 'schema',
         },
         {
           key: 'bookmark-6',
@@ -71,7 +71,7 @@ describe('MyBookmarks', () => {
           database: 'database',
           description: 'description',
           name: 'name',
-          schema_name: 'schema_name',
+          schema: 'schema',
         },
       ],
       isLoaded: true,

amundsen_application/static/js/components/common/ResourceListItem/TableListItem/index.tsx

@@ -21,13 +21,13 @@ class TableListItem extends React.Component<TableListItemProps, {}> {
 
   getDateLabel = () => {
     const { table } = this.props;
-    const dateTokens = new Date(table.last_updated_epoch * 1000).toDateString().split(' ');
+    const dateTokens = new Date(table.last_updated_timestamp * 1000).toDateString().split(' ');
     return `${dateTokens[1]} ${dateTokens[2]}, ${dateTokens[3]}`;
   };
 
   getLink = () => {
     const { table, logging } = this.props;
-    return `/table_detail/${table.cluster}/${table.database}/${table.schema_name}/${table.name}`
+    return `/table_detail/${table.cluster}/${table.database}/${table.schema}/${table.name}`
       + `?index=${logging.index}&source=${logging.source}`;
   };
 
@@ -37,7 +37,7 @@ class TableListItem extends React.Component<TableListItemProps, {}> {
 
   render() {
     const { table } = this.props;
-    const hasLastUpdated = !!table.last_updated_epoch;
+    const hasLastUpdated = !!table.last_updated_timestamp;
 
     return (
       <li className="list-group-item">
@@ -47,7 +47,7 @@ class TableListItem extends React.Component<TableListItemProps, {}> {
             <div className="resource-info-text">
               <div className="resource-name title-2">
                 <div className="truncated">
-                  { `${table.schema_name}.${table.name}`}
+                  { `${table.schema}.${table.name}`}
                 </div>
                 <BookmarkIcon bookmarkKey={ this.props.table.key }/>
               </div>

amundsen_application/static/js/components/common/ResourceListItem/TableListItem/tests/index.spec.tsx

@@ -31,9 +31,9 @@ describe('TableListItem', () => {
         database: 'testdb',
         description: 'I am the description',
         key: '',
-        last_updated_epoch: 1553829681,
+        last_updated_timestamp: 1553829681,
         name: 'tableName',
-        schema_name: 'tableSchema',
+        schema: 'tableSchema',
       },
       ...propOverrides
     };
@@ -53,7 +53,7 @@ describe('TableListItem', () => {
     it('getLink returns correct string', () => {
       const { props, wrapper } = setup();
       const { table, logging } = props;
-      expect(wrapper.instance().getLink()).toEqual(`/table_detail/${table.cluster}/${table.database}/${table.schema_name}/${table.name}?index=${logging.index}&source=${logging.source}`);
+      expect(wrapper.instance().getLink()).toEqual(`/table_detail/${table.cluster}/${table.database}/${table.schema}/${table.name}?index=${logging.index}&source=${logging.source}`);
     });
   });
 
@@ -133,7 +133,7 @@ describe('TableListItem', () => {
         expect(resourceBadges.exists()).toBe(true);
       });
 
-      describe('if props.table has last_updated_epoch', () => {
+      describe('if props.table has last_updated_timestamp', () => {
         it('renders Last Updated title', () => {
           expect(resourceBadges.children().at(0).children().at(0).text()).toEqual('Last Updated');
         });
@@ -143,7 +143,7 @@ describe('TableListItem', () => {
         });
       });
 
-      describe('if props.table does not have last_updated_epoch', () => {
+      describe('if props.table does not have last_updated_timestamp', () => {
         it('does not render Last Updated section', () => {
           const { props, wrapper } = setup({ table: {
             type: ResourceType.table,
@@ -151,9 +151,9 @@ describe('TableListItem', () => {
             database: '',
             description: 'I am the description',
             key: '',
-            last_updated_epoch: null,
+            last_updated_timestamp: null,
             name: 'tableName',
-            schema_name: 'tableSchema',
+            schema: 'tableSchema',
           }});
           expect(wrapper.find('.resource-badges').children()).toHaveLength(1);
         });

amundsen_application/static/js/components/common/SearchBar/InlineSearchResults/index.tsx

@@ -93,7 +93,7 @@ export class InlineSearchResults extends React.Component<InlineSearchResultsProp
     switch (resourceType) {
       case ResourceType.table:
         const table = result as TableResource;
-        return `/table_detail/${table.cluster}/${table.database}/${table.schema_name}/${table.name}?${logParams}`;
+        return `/table_detail/${table.cluster}/${table.database}/${table.schema}/${table.name}?${logParams}`;
       case ResourceType.user:
         const user = result as UserResource;
         return `/user/${user.user_id}?${logParams}`;
@@ -131,7 +131,7 @@ export class InlineSearchResults extends React.Component<InlineSearchResultsProp
     switch (resourceType) {
       case ResourceType.table:
         const table = result as TableResource;
-        return `${table.schema_name}.${table.name}`;
+        return `${table.schema}.${table.name}`;
       case ResourceType.user:
         const user = result as UserResource;
         return user.display_name;

amundsen_application/static/js/components/common/SearchBar/InlineSearchResults/tests/index.spec.tsx

@@ -183,9 +183,9 @@ describe('InlineSearchResults', () => {
     it('returns the correct href for ResourceType.table', () => {
       const index = 0;
       const givenTable = props.tables.results[index];
-      const { cluster, database, schema_name, name } = givenTable;
+      const { cluster, database, schema, name } = givenTable;
       const output = wrapper.instance().getSuggestedResultHref(ResourceType.table, givenTable, index);
-      expect(output).toEqual(`/table_detail/${cluster}/${database}/${schema_name}/${name}?source=inline_search&index=${index}`);
+      expect(output).toEqual(`/table_detail/${cluster}/${database}/${schema}/${name}?source=inline_search&index=${index}`);
     });
     it('returns the correct href for ResourceType.user', () => {
       const index = 0;
@@ -258,10 +258,10 @@ describe('InlineSearchResults', () => {
       props = setupResult.props;
       wrapper = setupResult.wrapper;
     });
-    it('returns the schema_name.name for ResourceType.table', () => {
+    it('returns the schema.name for ResourceType.table', () => {
       const givenTable = props.tables.results[0];
       const output = wrapper.instance().getSuggestedResultTitle(ResourceType.table, givenTable);
-      expect(output).toEqual(`${givenTable.schema_name}.${givenTable.name}`);
+      expect(output).toEqual(`${givenTable.schema}.${givenTable.name}`);
     });
     it('returns the display_name ResourceType.user', () => {
       const givenUser = props.users.results[0];

amundsen_application/static/js/ducks/bookmark/tests/index.spec.ts

@@ -42,7 +42,7 @@ describe('bookmark ducks', () => {
         database: 'database',
         description: 'description',
         name: 'name',
-        schema_name: 'schema_name',
+        schema: 'schema',
       },
     ];
   });
@@ -142,7 +142,7 @@ describe('bookmark ducks', () => {
           database: 'database',
           description: 'description',
           name: 'name',
-          schema_name: 'schema_name',
+          schema: 'schema',
         },
         {
           key: 'bookmarked_key_1',
@@ -151,7 +151,7 @@ describe('bookmark ducks', () => {
           database: 'database',
           description: 'description',
           name: 'name',
-          schema_name: 'schema_name',
+          schema: 'schema',
         },
       ];
       testState = {
@@ -182,7 +182,7 @@ describe('bookmark ducks', () => {
           database: 'database',
           description: 'description',
           name: 'name',
-          schema_name: 'schema_name',
+          schema: 'schema',
         }],
       });
     });

amundsen_application/static/js/ducks/search/tests/index.spec.ts

@@ -79,9 +79,9 @@ describe('search ducks', () => {
           database: 'testDatabase',
           description: 'I have a lot of users',
           key: 'testDatabase://testCluster.testSchema/testName',
-          last_updated_epoch: 946684799,
+          last_updated_timestamp: 946684799,
           name: 'testName',
-          schema_name: 'testSchema',
+          schema: 'testSchema',
           type: ResourceType.table,
         },
       ],
@@ -104,9 +104,9 @@ describe('search ducks', () => {
           database: 'testDatabase',
           description: 'I have a lot of users',
           key: 'testDatabase://testCluster.testSchema/testName',
-          last_updated_epoch: 946684799,
+          last_updated_timestamp: 946684799,
           name: 'testName',
-          schema_name: 'testSchema',
+          schema: 'testSchema',
           type: ResourceType.table,
         },
       ],

amundsen_application/static/js/ducks/tableMetadata/api/helpers.ts

@@ -45,8 +45,8 @@ export function createOwnerNotificationData(payload: UpdateOwnerPayload, tableDa
   return {
     notificationType: payload.method === UpdateMethod.PUT ? NotificationType.OWNER_ADDED : NotificationType.OWNER_REMOVED,
     options: {
-      resource_name: `${tableData.schema}.${tableData.table_name}`,
-      resource_path: `/table_detail/${tableData.cluster}/${tableData.database}/${tableData.schema}/${tableData.table_name}`
+      resource_name: `${tableData.schema}.${tableData.name}`,
+      resource_path: `/table_detail/${tableData.cluster}/${tableData.database}/${tableData.schema}/${tableData.name}`
     },
     recipients: [payload.id],
   };

amundsen_application/static/js/ducks/tableMetadata/api/tests/index.spec.ts

@@ -80,8 +80,8 @@ describe('helpers', () => {
     beforeAll(() => {
       testData = globalState.tableMetadata.tableData;
       testId =  'testId@test.com';
-      expectedName = `${testData.schema}.${testData.table_name}`;
-      expectedPath = `/table_detail/${testData.cluster}/${testData.database}/${testData.schema}/${testData.table_name}`;
+      expectedName = `${testData.schema}.${testData.name}`;
+      expectedPath = `/table_detail/${testData.cluster}/${testData.database}/${testData.schema}/${testData.name}`;
     });
 
     it('creates correct request data for PUT', () => {

amundsen_application/static/js/ducks/tableMetadata/api/v0.ts

@@ -62,7 +62,7 @@ export function getTableDescription(tableData: TableMetadata) {
   const tableParams = getTableQueryParams(tableData.key);
   return axios.get(`${API_PATH}/get_table_description?${tableParams}`)
   .then((response: AxiosResponse<DescriptionAPI>) => {
-    tableData.table_description = response.data.description;
+    tableData.description = response.data.description;
     return tableData;
   });
 }

amundsen_application/static/js/ducks/tableMetadata/reducer.ts

@@ -187,8 +187,8 @@ export const initialTableDataState: TableMetadata = {
   is_view: false,
   key: '',
   schema: '',
-  table_name: '',
-  table_description: '',
+  name: '',
+  description: '',
   table_writer: { application_url: '', description: '', id: '', name: '' },
   partition: { is_partitioned: false },
   table_readers: [],

amundsen_application/static/js/fixtures/globalState.ts

@@ -23,7 +23,7 @@ const globalState: GlobalState = {
         database: 'database',
         description: 'description',
         name: 'name',
-        schema_name: 'schema_name',
+        schema: 'schema',
       },
     ],
     myBookmarksIsLoaded: false,
@@ -43,7 +43,7 @@ const globalState: GlobalState = {
       description: 'I have a lot of users',
       key: 'testDatabase://testCluster.testSchema/testName',
       name: 'testName',
-      schema_name: 'testSchema',
+      schema: 'testSchema',
       type: ResourceType.table,
     },
     {
@@ -52,7 +52,7 @@ const globalState: GlobalState = {
       description: 'I also have a lot of users',
       key: 'testDatabase://testCluster.testSchema/otherName',
       name: 'otherName',
-      schema_name: 'testSchema',
+      schema: 'testSchema',
       type: ResourceType.table,
     }
   ],
@@ -73,9 +73,9 @@ const globalState: GlobalState = {
           database: 'testDatabase',
           description: 'I have a lot of users',
           key: 'testDatabase://testCluster.testSchema/testName',
-          last_updated_epoch: 946684799,
+          last_updated_timestamp: 946684799,
           name: 'testName',
-          schema_name: 'testSchema',
+          schema: 'testSchema',
           type: ResourceType.table,
         },
       ],
@@ -117,8 +117,8 @@ const globalState: GlobalState = {
       is_view: false,
       key: '',
       schema: '',
-      table_name: '',
-      table_description: '',
+      name: '',
+      description: '',
       table_writer: { application_url: '', description: '', id: '', name: '' },
       partition: { is_partitioned: false },
       table_readers: [],

amundsen_application/static/js/fixtures/search/inlineResults.ts

@@ -38,9 +38,9 @@ export const allResourcesExample = {
         database: 'testDatabase',
         description: 'I have a lot of users',
         key: 'testDatabase://testCluster.testSchema/testName1',
-        last_updated_epoch: 946684799,
+        last_updated_timestamp: 946684799,
         name: 'testName1',
-        schema_name: 'testSchema',
+        schema: 'testSchema',
         type: ResourceType.table,
       },
       {
@@ -48,9 +48,9 @@ export const allResourcesExample = {
         database: 'testDatabase',
         description: 'I have a lot of users',
         key: 'testDatabase://testCluster.testSchema/testName2',
-        last_updated_epoch: 946684799,
+        last_updated_timestamp: 946684799,
         name: 'testName2',
-        schema_name: 'testSchema',
+        schema: 'testSchema',
         type: ResourceType.table,
       },
       {
@@ -58,9 +58,9 @@ export const allResourcesExample = {
         database: 'testDatabase',
         description: 'I have a lot of users',
         key: 'testDatabase://testCluster.testSchema/testName3',
-        last_updated_epoch: 946684799,
+        last_updated_timestamp: 946684799,
         name: 'testName3',
-        schema_name: 'testSchema',
+        schema: 'testSchema',
         type: ResourceType.table,
       }
     ],

amundsen_application/static/js/interfaces/Resources.ts

@@ -24,10 +24,10 @@ export interface TableResource extends Resource {
   database: string;
   description: string;
   key: string;
-  // 'popular_tables' currently does not support 'last_updated_epoch'
-  last_updated_epoch?: number;
+  // 'popular_tables' currently does not support 'last_updated_timestamp'
+  last_updated_timestamp?: number;
   name: string;
-  schema_name: string;
+  schema: string;
 };
 
 export interface UserResource extends Resource, PeopleUser {

amundsen_application/static/js/interfaces/TableMetadata.ts

@@ -28,7 +28,7 @@ export interface TableColumnStats {
 
 export interface TableReader {
   read_count: number;
-  reader: User;
+  user: User;
 }
 
 export interface TableSource {
@@ -59,7 +59,7 @@ export interface TableColumn {
   name: string;
   description: string;
   is_editable: boolean;
-  type: string;
+  col_type: string;
   stats: TableColumnStats[];
 }
 
@@ -77,8 +77,8 @@ export interface TableMetadata {
   is_view: boolean;
   key: string;
   schema: string;
-  table_name: string;
-  table_description: string;
+  name: string;
+  description: string;
   table_writer: TableWriter;
   partition: PartitionData;
   table_readers: TableReader[];

requirements.txt

@@ -58,7 +58,14 @@ requests==2.20.0
 # Upstream url: https://github.com/marshmallow-code/marshmallow
 marshmallow==2.15.3
 
+# Allows declaring marshmallow schema through type annotations
+# License: MIT
+# Upstream url: https://github.com/justanr/marshmallow-annotations
+marshmallow-annotations==2.4.0
+
 # A utility library for mocking out the requests Python library.
 # License: Apache 2.0
 # Upstream url: https://pypi.org/project/responses/
 responses==0.9.0
+
+amundsen-common==0.2.2
\ No newline at end of file

setup.py

@@ -34,7 +34,7 @@ requirements_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'r
 with open(requirements_path) as requirements_file:
     requirements = requirements_file.readlines()
 
-__version__ = '1.2.1'
+__version__ = '2.0.0'
 
 
 setup(

tests/unit/api/metadata/test_v0.py

@@ -20,11 +20,10 @@ class MetadataTest(unittest.TestCase):
                 {
                     'cluster': 'test_cluster',
                     'database': 'test_db',
-                    'key': 'test_db://test_cluster.test_schema/test_table',
                     'schema': 'test_schema',
-                    'table_description': 'This is a test',
-                    'table_name': 'test_table',
-                    'type': 'table',
+                    'description': 'This is a test',
+                    'name': 'test_table',
+                    'key': 'test_db://test_cluster.test_schema/test_table',
                 }
             ]
         }
@@ -33,11 +32,11 @@ class MetadataTest(unittest.TestCase):
                 'cluster': 'test_cluster',
                 'database': 'test_db',
                 'description': 'This is a test',
-                'key': 'test_db://test_cluster.test_schema/test_table',
-                'schema_name': 'test_schema',
+                'schema': 'test_schema',
                 'type': 'table',
                 'name': 'test_table',
-                'last_updated_epoch': None,
+                'key': 'test_db://test_cluster.test_schema/test_table',
+                'last_updated_timestamp': None,
             }
         ]
         self.mock_metadata = {
@@ -46,7 +45,7 @@ class MetadataTest(unittest.TestCase):
                 {
                     'name': 'column_1',
                     'description': 'This is a test',
-                    'type': 'bigint',
+                    'col_type': 'bigint',
                     'sort_order': 0,
                     'stats': [
                         {'stat_type': 'count', 'stat_val': '100', 'start_epoch': 1538352000, 'end_epoch': 1538352000},
@@ -59,11 +58,12 @@ class MetadataTest(unittest.TestCase):
             'key': 'test_db://test_cluster.test_schema/test_table',
             'owners': [],
             'schema': 'test_schema',
-            'table_name': 'test_table',
-            'table_description': 'This is a test',
+            'name': 'test_table',
+            'description': 'This is a test',
+            'programmatic_descriptions': [],
             'tags': [],
             'table_readers': [
-                {'reader': {'email': 'test@test.com', 'first_name': None, 'last_name': None}, 'read_count': 100}
+                {'user': {'email': 'test@test.com', 'first_name': None, 'last_name': None}, 'read_count': 100}
             ],
             'watermarks': [
                 {'watermark_type': 'low_watermark', 'partition_key': 'ds', 'partition_value': '', 'create_time': ''},
@@ -77,17 +77,18 @@ class MetadataTest(unittest.TestCase):
             },
         }
         self.expected_parsed_metadata = {
-            'key': 'table_key',
             'badges': [],
             'cluster': 'test_cluster',
             'database': 'test_db',
             'schema': 'test_schema',
-            'table_name': 'test_table',
-            'table_description': 'This is a test',
+            'name': 'test_table',
+            'key': 'test_db://test_cluster.test_schema/test_table',
+            'description': 'This is a test',
+            'programmatic_descriptions': [],
             'tags': [],
             'table_readers': [
                 {
-                    'reader': {
+                    'user': {
                         'email': 'test@test.com',
                         'first_name': None,
                         'last_name': None,
@@ -108,7 +109,7 @@ class MetadataTest(unittest.TestCase):
                 {
                     'name': 'column_1',
                     'description': 'This is a test',
-                    'type': 'bigint',
+                    'col_type': 'bigint',
                     'sort_order': 0,
                     'stats': [
                         {'stat_type': 'count', 'stat_val': '100', 'start_epoch': 1538352000, 'end_epoch': 1538352000},
@@ -128,7 +129,8 @@ class MetadataTest(unittest.TestCase):
                 {'watermark_type': 'high_watermark', 'partition_key': 'ds', 'partition_value': '', 'create_time': ''}
             ],
             'source': '/source',
-            'is_editable': True
+            'is_editable': True,
+            'last_updated_timestamp': None,
         }
         self.mock_tags = {
             'tag_usages': [
@@ -209,15 +211,15 @@ class MetadataTest(unittest.TestCase):
                     'cluster': 'cluster',
                     'database': 'database',
                     'schema': 'schema',
-                    'table_name': 'table_name_0',
-                    'table_description': 'description',
+                    'name': 'table_name_0',
+                    'description': 'description',
                 },
                 {
                     'cluster': 'cluster',
                     'database': 'database',
                     'schema': 'schema',
-                    'table_name': 'table_name_1',
-                    'table_description': 'description',
+                    'name': 'table_name_1',
+                    'description': 'description',
                 },
             ]
         }
@@ -226,21 +228,21 @@ class MetadataTest(unittest.TestCase):
                 'cluster': 'cluster',
                 'database': 'database',
                 'description': 'description',
-                'key': 'database://cluster.schema/table_name_0',
-                'last_updated_epoch': None,
+                'last_updated_timestamp': None,
                 'name': 'table_name_0',
-                'schema_name': 'schema',
+                'schema': 'schema',
                 'type': 'table',
+                'key': 'database://cluster.schema/table_name_0',
             },
             {
                 'cluster': 'cluster',
                 'database': 'database',
                 'description': 'description',
-                'key': 'database://cluster.schema/table_name_1',
-                'last_updated_epoch': None,
+                'last_updated_timestamp': None,
                 'name': 'table_name_1',
-                'schema_name': 'schema',
+                'schema': 'schema',
                 'type': 'table',
+                'key': 'database://cluster.schema/table_name_1',
             },
         ]
 

tests/unit/api/search/test_v0.py

@@ -25,9 +25,9 @@ class SearchTest(unittest.TestCase):
                     'database': 'test_db',
                     'description': 'This is a test',
                     'key': 'test_key',
-                    'last_updated_epoch': 1527283287,
+                    'last_updated_timestamp': 1527283287,
                     'name': 'test_table',
-                    'schema_name': 'test_schema',
+                    'schema': 'test_schema',
                     'tags': [],
                 }
             ]
@@ -39,9 +39,9 @@ class SearchTest(unittest.TestCase):
                 'database': 'test_db',
                 'description': 'This is a test',
                 'key': 'test_key',
-                'last_updated_epoch': 1527283287,
+                'last_updated_timestamp': 1527283287,
                 'name': 'test_table',
-                'schema_name': 'test_schema',
+                'schema': 'test_schema',
             }
         ]
         self.mock_search_user_results = {