Added action logging for frontend events (#59)

* Added a generic action logging API 'log_event'
* Added a scroll tracker on the table details page
* Added logging events to navigation links, tags, and other actions.

amundsen_application/__init__.py

@@ -8,6 +8,7 @@ from flask import Flask
 from amundsen_application.api import init_routes
 from amundsen_application.api.v0 import blueprint
 from amundsen_application.api.announcements.v0 import announcements_blueprint
+from amundsen_application.api.log.v0 import log_blueprint
 from amundsen_application.api.mail.v0 import mail_blueprint
 from amundsen_application.api.metadata.v0 import metadata_blueprint
 from amundsen_application.api.preview.v0 import preview_blueprint
@@ -46,6 +47,7 @@ def create_app(config_module_class: str, template_folder: str = None) -> Flask:
 
     app.register_blueprint(blueprint)
     app.register_blueprint(announcements_blueprint)
+    app.register_blueprint(log_blueprint)
     app.register_blueprint(mail_blueprint)
     app.register_blueprint(metadata_blueprint)
     app.register_blueprint(preview_blueprint)

amundsen_application/api/log/v0.py

+import logging
+
+from http import HTTPStatus
+
+from flask import Response, jsonify, make_response, request
+from flask.blueprints import Blueprint
+
+from amundsen_application.log.action_log import action_logging
+from amundsen_application.api.utils.request_utils import get_query_param
+
+
+LOGGER = logging.getLogger(__name__)
+
+log_blueprint = Blueprint('log', __name__, url_prefix='/api/log/v0')
+
+
+@log_blueprint.route('/log_event', methods=['POST'])
+def log_generic_action() -> Response:
+    """
+    Log a generic action on the frontend. Captured parameters include
+
+    :param command: Req. User Action E.g. click, scroll, hover, search, etc
+    :param target_id: Req. Unique identifier for the object acted upon E.g. tag::payments, table::schema.database
+    :param target_type: Opt. Type of element event took place on (button, link, tag, icon, etc)
+    :param label: Opt. Displayed text for target
+    :param location: Opt. Where the the event occurred
+    :param value: Opt. Value to be logged
+    :return:
+    """
+    @action_logging
+    def _log_generic_action(*,
+                            command: str,
+                            target_id: str,
+                            target_type: str,
+                            label: str,
+                            location: str,
+                            value: str) -> None:
+        pass  # pragma: no cover
+
+    try:
+        args = request.get_json()
+        command = get_query_param(args, 'command', '"command" is a required parameter.')
+        target_id = get_query_param(args, 'target_id', '"target_id" is a required field.')
+        _log_generic_action(
+            command=command,
+            target_id=target_id,
+            target_type=args.get('target_type', None),
+            label=args.get('label', None),
+            location=args.get('location', None),
+            value=args.get('value', None)
+        )
+        message = 'Logging of {} action successful'.format(command)
+        return make_response(jsonify({'msg': message}), HTTPStatus.OK)
+
+    except Exception as e:
+        message = 'Log action failed. Encountered exception: ' + str(e)
+        logging.exception(message)
+        payload = jsonify({'msg': message})
+        return make_response(payload, HTTPStatus.INTERNAL_SERVER_ERROR)

amundsen_application/log/action_log.py

@@ -70,11 +70,13 @@ def _build_metrics(func_name: str,
     :return: Dict that matches ActionLogParams variable
     """
 
-    metrics = {'command': func_name}  # type: Dict[str, Any]
-    metrics['start_epoch_ms'] = get_epoch_millisec()
-    metrics['host_name'] = socket.gethostname()
-    metrics['pos_args_json'] = json.dumps(args)
-    metrics['keyword_args_json'] = json.dumps(kwargs)
+    metrics = {
+        'command': kwargs.get('command', func_name),
+        'start_epoch_ms': get_epoch_millisec(),
+        'host_name': socket.gethostname(),
+        'pos_args_json': json.dumps(args),
+        'keyword_args_json': json.dumps(kwargs),
+    }  # type: Dict[str, Any]
 
     if flask_app.config['AUTH_USER_METHOD']:
         metrics['user'] = flask_app.config['AUTH_USER_METHOD'](flask_app).email

amundsen_application/static/js/components/NavBar/index.tsx

@@ -8,6 +8,7 @@ import AppConfig from 'config/config';
 import { GlobalState } from 'ducks/rootReducer';
 import { getLoggedInUser } from 'ducks/user/reducer';
 import { LoggedInUser, GetLoggedInUserRequest } from 'ducks/user/types';
+import { logClick } from "ducks/utilMethods";
 
 import './styles.scss';
 
@@ -63,9 +64,17 @@ export class NavBar extends React.Component<NavBarProps, NavBarState> {
               {
                 AppConfig.navLinks.map((link, index) => {
                   if (link.use_router) {
-                    return <NavLink key={index} to={link.href} target={link.target}>{link.label}</NavLink>
+                    return (
+                      <NavLink id={ link.id } key={ index } to={ link.href } target={ link.target } onClick={logClick}>
+                        {link.label}
+                      </NavLink>
+                    )
                   }
-                  return <a key={index} href={link.href} target={link.target}>{link.label}</a>
+                  return (
+                    <a id={ link.id } key={ index } href={ link.href } target={ link.target } onClick={logClick}>
+                      {link.label}
+                    </a>
+                  )
                 })
               }
               {

amundsen_application/static/js/components/TableDetail/ColumnDescEditableText/index.tsx

@@ -19,10 +19,10 @@ export const mapStateToProps = (state: GlobalState, ownProps: ContainerOwnProps)
 export const mapDispatchToProps = (dispatch: any, ownProps: ContainerOwnProps) => {
   const getLatestValue = function(onSuccess, onFailure) {
     return getColumnDescription(ownProps.columnIndex, onSuccess, onFailure);
-  }
+  };
   const onSubmitValue = function(newValue, onSuccess, onFailure) {
     return updateColumnDescription(newValue, ownProps.columnIndex, onSuccess, onFailure);
-  }
+  };
 
   return bindActionCreators({ getLatestValue, onSubmitValue } , dispatch);
 };

amundsen_application/static/js/components/TableDetail/DataPreviewButton/index.tsx

@@ -6,6 +6,7 @@ import Linkify from 'react-linkify'
 
 import { GlobalState } from 'ducks/rootReducer';
 import { PreviewData } from '../types';
+import { logClick } from 'ducks/utilMethods';
 
 // TODO: Use css-modules instead of 'import'
 import './styles.scss';
@@ -77,11 +78,12 @@ export class DataPreviewButton extends React.Component<DataPreviewButtonProps, D
 
   handleClose = () => {
    this.setState({ showModal: false });
-  }
+  };
 
-  handleShow = () => {
+  handleClick = (e) => {
+    logClick(e);
     this.setState({ showModal: true });
-  }
+  };
 
   getSanitizedValue(value) {
     // Display the string interpretation of the following "false-y" values
@@ -183,9 +185,11 @@ export class DataPreviewButton extends React.Component<DataPreviewButtonProps, D
 
     const previewButton = (
       <button
+        id="data-preview-button"
         className="btn btn-default btn-block"
         disabled={disabled}
-       onClick={this.handleShow}>
+        onClick={this.handleClick}
+      >
          <img className={"icon icon-color " + iconClass} />
          <span>{buttonText}</span>
       </button>

amundsen_application/static/js/components/TableDetail/DetailList/DetailListItem/index.tsx

@@ -3,6 +3,7 @@ import moment from 'moment-timezone';
 
 import ColumnDescEditableText from 'components/TableDetail/ColumnDescEditableText';
 import { TableColumn } from 'components/TableDetail/types';
+import { logClick } from 'ducks/utilMethods';
 
 // TODO: Use css-modules instead of 'import'
 import './styles.scss';
@@ -29,7 +30,16 @@ class DetailListItem extends React.Component<DetailListItemProps, DetailListItem
     };
   }
 
-  onClick = () => {
+  onClick = (e) => {
+    if (!this.state.isExpanded) {
+      const metadata = this.props.data;
+      logClick(e, {
+        target_id: `column::${metadata.name}`,
+        target_type: 'column stats',
+        label: `${metadata.name} ${metadata.type}`,
+      });
+    }
+
     this.setState(prevState => ({
       isExpanded: !prevState.isExpanded
     }));

amundsen_application/static/js/components/TableDetail/index.tsx

@@ -14,6 +14,7 @@ import AvatarLabel from 'components/common/AvatarLabel';
 import Breadcrumb from 'components/common/Breadcrumb';
 import EntityCard from 'components/common/EntityCard';
 import LoadingSpinner from 'components/common/LoadingSpinner';
+import ScrollTracker from "components/common/ScrollTracker";
 import TagInput from 'components/Tags/TagInput';
 
 import DataPreviewButton from './DataPreviewButton';
@@ -21,12 +22,13 @@ import DetailList from './DetailList';
 import OwnerEditor from './OwnerEditor';
 import TableDescEditableText from './TableDescEditableText';
 import WatermarkLabel from "./WatermarkLabel";
+import { logClick } from 'ducks/utilMethods';
 
 import Avatar from 'react-avatar';
 import { OverlayTrigger, Popover } from 'react-bootstrap';
 import { RouteComponentProps } from 'react-router';
 
-import { PreviewQueryParams, TableMetadata, TableOwners } from './types';
+import { PreviewQueryParams, TableMetadata } from './types';
 
 // TODO: Use css-modules instead of 'import'
 import './styles.scss';
@@ -55,6 +57,7 @@ export class TableDetail extends React.Component<TableDetailProps & RouteCompone
   private database: string;
   private schema: string;
   private tableName: string;
+  private displayName: string;
   public static defaultProps: TableDetailProps = {
     getTableData: () => undefined,
     getPreviewData: () => undefined,
@@ -90,6 +93,7 @@ export class TableDetail extends React.Component<TableDetailProps & RouteCompone
     this.database = params ? params.db : '';
     this.schema = params ? params.schema : '';
     this.tableName = params ? params.table : '';
+    this.displayName = params ? `${this.schema}.${this.tableName}` : '';
 
     this.state = {
       isLoading: props.isLoading,
@@ -112,6 +116,12 @@ export class TableDetail extends React.Component<TableDetailProps & RouteCompone
     this.props.getPreviewData({ schema: this.schema, tableName: this.tableName });
   }
 
+  frequentUserOnClick = (e) => {
+    logClick(e, {
+      target_id: 'frequent-users',
+    })
+  };
+
   getAvatarForUser(fullName, profileUrl) {
     const popoverHoverFocus = (
       <Popover id="popover-trigger-hover-focus">
@@ -121,8 +131,10 @@ export class TableDetail extends React.Component<TableDetailProps & RouteCompone
     if (profileUrl.length !== 0) {
       return (
         <OverlayTrigger key={fullName} trigger={['hover', 'focus']} placement="top" overlay={popoverHoverFocus}>
-          <a href={profileUrl} target='_blank' style={{ display: 'inline-block', marginLeft: '-5px',
-            backgroundColor: 'white', borderRadius: '90%'}}>
+          <a href={profileUrl} target='_blank'
+             style={{ display: 'inline-block', marginLeft: '-5px', backgroundColor: 'white', borderRadius: '90%'}}
+             onClick={this.frequentUserOnClick}
+          >
             <Avatar name={fullName} size={25} round={true} style={{ border: '1px solid white' }} />
           </a>
         </OverlayTrigger>
@@ -147,7 +159,7 @@ export class TableDetail extends React.Component<TableDetailProps & RouteCompone
 
     if (appUrl.length !== 0) {
       return (
-        <a href={appUrl} target='_blank'>
+        <a href={appUrl} target='_blank' id="explore-writer" onClick={logClick}>
           { avatarLabel }
         </a>
       );
@@ -156,27 +168,28 @@ export class TableDetail extends React.Component<TableDetailProps & RouteCompone
     return avatarLabel;
   }
 
-  getAvatarForTableSource(schema, table, source) {
-
+  getAvatarForTableSource = (source) => {
     if (source !== null) {
       const image = (source.source_type === 'github')? '/static/images/github.png': '';
-      const displayName = schema + '.' + table;
-      const avatarLabel = <AvatarLabel label={displayName} src={image}/>;
+      const avatarLabel = <AvatarLabel label={this.displayName} src={image}/>;
 
       return (
-        <a href={source.source} target='_blank'>
+        <a href={ source.source }
+           target='_blank'
+           id="explore-source"
+           onClick={ logClick }
+        >
           { avatarLabel }
         </a>
       );
     }
-  }
+  };
 
   getAvatarForLineage = () => {
     const href = AppConfig.tableLineage.urlGenerator(this.database, this.cluster, this.schema, this.tableName);
-    const displayName = `${this.schema}.${this.tableName}`;
     return (
-      <a href={ href } target='_blank'>
-        <AvatarLabel label={ displayName } src={ AppConfig.tableLineage.iconPath }/>
+      <a href={ href } target='_blank' id="explore-lineage" onClick={logClick}>
+        <AvatarLabel label={ this.displayName } src={ AppConfig.tableLineage.iconPath }/>
       </a>
     );
   };
@@ -240,7 +253,7 @@ export class TableDetail extends React.Component<TableDetailProps & RouteCompone
     const source = data.source;
     if (source && source.source !== null) {
       const sourceRenderer = () => {
-        return this.getAvatarForTableSource(data.schema, data.table_name, source);
+        return this.getAvatarForTableSource(source);
       };
 
       entityCardSections.push({'title': 'Source Code', 'contentRenderer': sourceRenderer, 'isEditable': false});
@@ -259,13 +272,16 @@ export class TableDetail extends React.Component<TableDetailProps & RouteCompone
     const previewSectionRenderer = () => {
       return (
         <div>
-          <DataPreviewButton modalTitle={`${this.schema}.${this.tableName}`} />
+          <DataPreviewButton modalTitle={ this.displayName } />
           {
             AppConfig.tableProfile.isExploreEnabled &&
-              <a className="btn btn-default btn-block"
+              <a
+                className="btn btn-default btn-block"
                 href={this.getExploreSqlUrl()}
                 role="button"
                 target="_blank"
+                id="explore-sql"
+                onClick={logClick}
               >
                 <img className="icon icon-color icon-database"/>
                 Explore with SQL
@@ -292,7 +308,7 @@ export class TableDetail extends React.Component<TableDetailProps & RouteCompone
 
 
     return entityCardSections;
-  }
+  };
 
   render() {
     const data = this.state.tableData;
@@ -332,16 +348,17 @@ export class TableDetail extends React.Component<TableDetailProps & RouteCompone
                 />
               </div>
             </div>
+            <ScrollTracker targetId={ this.displayName }/>
           </div>
         );
     }
     return (
-      <DocumentTitle title={ `${this.schema}.${this.tableName} - Amundsen Table Details` }>
+      <DocumentTitle title={ `${this.displayName} - Amundsen Table Details` }>
         { innerContent }
       </DocumentTitle>
     );
   }
-};
+}
 
 export const mapStateToProps = (state: GlobalState) => {
   return {

amundsen_application/static/js/components/Tags/TagInfo/index.tsx

 import * as React from 'react';
 import { Link } from 'react-router-dom';
 import { Tag } from '../types';
+import { logClick } from 'ducks/utilMethods';
 
 import './styles.scss';
 
@@ -18,21 +19,30 @@ class TagInfo extends React.Component<TagInfoProps, {}> {
     super(props);
   }
 
+  onClick = (e) => {
+    logClick(e, {
+      target_type: 'tag',
+      label: this.props.data.tag_name,
+    });
+  };
+
   render() {
-    const searchUrl = `/search?searchTerm=tag:${this.props.data.tag_name}`;
+    const name = this.props.data.tag_name;
+    const searchUrl = `/search?searchTerm=tag:${name}`;
 
-    if (this.props.compact) {
     return (
-        <Link role="button" to={searchUrl} className="btn tag-button compact">
-          {this.props.data.tag_name}
-        </Link>
-      );
+      <Link
+        id={ `tag::${name}` }
+        role="button"
+        to={ searchUrl }
+        className={ "btn tag-button" + (this.props.compact ? " compact" : "") }
+        onClick={ this.onClick }
+      >
+        <span className="tag-name">{ name }</span>
+        {
+          !this.props.compact &&
+            <span className="tag-count">{ this.props.data.tag_count }</span>
         }
-
-    return (
-      <Link role="button" to={searchUrl} className="btn tag-button">
-        <span className="tag-name">{this.props.data.tag_name}</span>
-        <span className="tag-count">{this.props.data.tag_count}</span>
       </Link>
     );
   }

amundsen_application/static/js/components/Tags/TagInput/index.tsx

 import * as React from 'react';
 import { connect } from 'react-redux';
 import { bindActionCreators } from 'redux';
-
-import ReactDOM from 'react-dom';
 import { Modal } from 'react-bootstrap';
-import Select, { components } from 'react-select';
+import { components } from 'react-select';
 import CreatableSelect from 'react-select/lib/Creatable';
-import makeAnimated from 'react-select/lib/animated';
 
 import { GlobalState } from 'ducks/rootReducer';
 import { getAllTags } from 'ducks/allTags/reducer';
@@ -115,7 +112,7 @@ class TagInput extends React.Component<TagInputProps, TagInputState> {
     }, []);
     this.props.updateTags(tagArray);
     this.handleClose();
-  }
+  };
 
   generateCustomOptionStyle(provided, state) {
     // https://react-select.com/props#api
@@ -168,7 +165,7 @@ class TagInput extends React.Component<TagInputProps, TagInputState> {
       tag = actionPayload.removedValue.value;
       this.props.updateTags([{'methodName': UpdateTagMethod.DELETE, 'tagName': tag}]);
     }
-  }
+  };
 
   preventDeleteOnBackSpace(event) {
     if (event.keyCode === 8 && event.target.value.length === 0){
@@ -236,7 +233,7 @@ class TagInput extends React.Component<TagInputProps, TagInputState> {
     } : {
       DropdownIndicator: () => { return null },
       IndicatorSeparator: () => { return null },
-    } ;
+    };
 
     let tagBody;
     if (this.state.readOnly) {

amundsen_application/static/js/components/common/ScrollTracker/index.tsx

+import * as React from 'react';
+import { throttle } from 'throttle-debounce';
+import { logAction } from 'ducks/utilMethods';
+
+
+export interface ScrollTrackerProps {
+  targetId: string;
+}
+
+interface ScrollTrackerState {
+  thresholds: number[];
+}
+
+export default class ScrollTracker extends React.Component<ScrollTrackerProps, ScrollTrackerState> {
+  private readonly throttledScroll: any;
+
+  constructor(props) {
+    super(props);
+
+    this.state = {
+      thresholds: [25, 50, 75, 100]
+    };
+
+    this.throttledScroll = throttle(100, false, this.onScroll);
+  }
+
+  componentDidMount() {
+    window.addEventListener("scroll", this.throttledScroll);
+  }
+
+  componentWillUnmount() {
+    window.removeEventListener("scroll", this.throttledScroll);
+  }
+
+  onScroll = () => {
+    if (this.state.thresholds.length === 0) {
+      window.removeEventListener("scroll", this.throttledScroll);
+      return;
+    }
+    const threshold = this.state.thresholds[0];
+    const scrollTop = window.pageYOffset || document.documentElement.scrollTop || document.body.scrollTop || 0;
+    const windowHeight = window.innerHeight || document.body.clientHeight;
+    const contentHeight = document.body.offsetHeight;
+    const scrollableAmount = Math.max(contentHeight - windowHeight, 1);
+
+    if (threshold <= 100 * scrollTop / scrollableAmount) {
+      this.fireAnalyticsEvent(this.state.thresholds.shift());
+    }
+  };
+
+  fireAnalyticsEvent = (threshold: number) => {
+    logAction({
+      command: "scroll",
+      target_id: this.props.targetId,
+      value: threshold.toString(),
+    });
+  };
+
+  render() { return null; }
+}
+

amundsen_application/static/js/config/config-default.ts

@@ -13,11 +13,13 @@ const configDefault: AppConfig = {
   navLinks: [
     {
       label: "Announcements",
+      id: "nav::announcements",
       href: "/announcements",
       use_router: true,
     },
     {
       label: "Browse",
+      id: "nav::browse",
       href: "/browse",
       use_router: true,
     }

amundsen_application/static/js/config/config-types.ts

@@ -74,6 +74,7 @@ interface TableLineageConfig {
 
 interface LinkConfig {
   href: string;
+  id: string;
   label: string;
   target?: string;
   use_router: boolean;

amundsen_application/static/js/ducks/log/api/v0.ts

+import axios, { AxiosResponse, AxiosError } from 'axios';
+
+export interface ActionLogParams {
+  command?: string;
+  target_id?: string;
+  target_type?: string;
+  label?: string;
+  location?: string;
+  value?: string;
+}
+
+const BASE_URL = '/api/log/v0/log_event';
+
+
+export function postActionLog(params: ActionLogParams) {
+  axios.post(BASE_URL, params)
+  .then((response: AxiosResponse) => {
+    return response.data;
+  })
+  .catch((error: AxiosError) => {
+    if (error.response) {
+      return error.response.data;
+    }
+  });
+}

amundsen_application/static/js/ducks/utilMethods.ts

+import { ActionLogParams, postActionLog } from "./log/api/v0";
 import { Tag } from 'components/Tags/types';
 
 export function sortTagsAlphabetical(a: Tag, b: Tag): number {
@@ -25,3 +26,35 @@ export function filterFromObj(initialObj: object, rejectedKeys: string[]): objec
     return obj;
   }, {});
 }
+
+export function logClick(event: React.MouseEvent<HTMLElement>, declaredProps?: ActionLogParams) {
+  const target = event.currentTarget;
+  const inferredProps: ActionLogParams = {
+    command: "click",
+    target_id: target.id,
+    label: target.innerText || target.textContent,
+  };
+
+  if (target.nodeValue !== null) {
+    inferredProps.value = target.nodeValue
+  }
+
+  let nodeName = target.nodeName.toLowerCase();
+  if (nodeName === 'a') {
+    if (target.classList.contains('btn')) {
+      nodeName = 'button';
+    } else {
+      nodeName = 'link';
+    }
+  }
+  inferredProps.target_type = nodeName;
+
+  logAction({ ...inferredProps, ...declaredProps });
+}
+
+export function logAction(declaredProps: ActionLogParams) {
+  const inferredProps = {
+    location: window.location.pathname
+  };
+  postActionLog({ ...inferredProps, ...declaredProps });
+}

amundsen_application/static/package-lock.json

@@ -14082,6 +14082,11 @@
       "integrity": "sha1-iQN8vJLFarGJJua6TLsgDhVnKmo=",
       "dev": true
     },
+    "throttle-debounce": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/throttle-debounce/-/throttle-debounce-2.1.0.tgz",
+      "integrity": "sha512-AOvyNahXQuU7NN+VVvOOX+uW6FPaWdAOdRP5HfwYxAfCzXTFKRMoIMk+n+po318+ktcChx+F1Dd91G3YHeMKyg=="
+    },
     "through": {
       "version": "2.3.8",
       "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz",

amundsen_application/static/package.json

@@ -97,6 +97,7 @@
     "redux-promise": "^0.5.3",
     "sanitize-html": "^1.16.1",
     "simple-query-string": "^1.3.2",
+    "throttle-debounce": "^2.1.0",
     "urijs": "^1.19.1"
   },
   "optionalDependencies": {