feat: Integration with Dremio (#859)

* Added Dremio preview example
Signed-off-by: Josh Howard <joshthoward@gmail.com>

* Checkpoint on flight preview
Signed-off-by: Josh Howard <joshthoward@gmail.com>

* Reworked flight preview to construct new client on each invocation; somehow reduces loading times
Signed-off-by: Josh Howard <joshthoward@gmail.com>

* Updated table icon and sources link to include Dremio
Signed-off-by: Josh Howard <joshthoward@gmail.com>

* Fixed flake errors
Signed-off-by: Josh Howard <joshthoward@gmail.com>

* Fixed code review comments
Signed-off-by: Josh Howard <joshthoward@gmail.com>

amundsen_application/base/examples/example_dremio_preview_client.py

+# Copyright Contributors to the Amundsen project.
+# SPDX-License-Identifier: Apache-2.0
+
+from http import HTTPStatus
+import logging
+from typing import Dict  # noqa: F401
+
+from flask import Response, jsonify, make_response, current_app as app
+from pyarrow import flight
+
+from amundsen_application.base.base_superset_preview_client import BasePreviewClient
+from amundsen_application.models.preview_data import PreviewData, PreviewDataSchema, ColumnItem
+
+
+class _DremioAuthHandler(flight.ClientAuthHandler):
+    """ClientAuthHandler for connections to Dremio server endpoint.
+    """
+    def __init__(self, username: str, password: str) -> None:
+        self.username = username
+        self.password = password
+        super(flight.ClientAuthHandler, self).__init__()
+
+    def authenticate(self, outgoing: flight.ClientAuthSender,
+                     incoming: flight.ClientAuthReader) -> None:
+        """Authenticate with Dremio user credentials.
+        """
+        basic_auth = flight.BasicAuth(self.username, self.password)
+        outgoing.write(basic_auth.serialize())
+        self.token = incoming.read()
+
+    def get_token(self,) -> str:
+        """Get the token from this AuthHandler.
+        """
+        return self.token
+
+
+class DremioPreviewClient(BasePreviewClient):
+
+    SQL_STATEMENT = 'SELECT * FROM {schema}."{table}" LIMIT 50'
+
+    def __init__(self,) -> None:
+        self.url = app.config['PREVIEW_CLIENT_URL']
+        self.username = app.config['PREVIEW_CLIENT_USERNAME']
+        self.password = app.config['PREVIEW_CLIENT_PASSWORD']
+
+        self.connection_args: Dict[str, bytes] = {}
+        tls_root_certs_path = app.config['PREVIEW_CLIENT_CERTIFICATE']
+        if tls_root_certs_path is not None:
+            with open(tls_root_certs_path, "rb") as f:
+                self.connection_args["tls_root_certs"] = f.read()
+
+    def get_preview_data(self, params: Dict, optionalHeaders: Dict = None) -> Response:
+        """Preview data from Dremio source
+        """
+        database = params.get('database')
+        if database != 'DREMIO':
+            logging.info('Skipping table preview for non-Dremio table')
+            return make_response(jsonify({'preview_data': {}}), HTTPStatus.OK)
+
+        try:
+            # Format base SQL_STATEMENT with request table and schema
+            schema = '"{}"'.format(params['schema'].replace('.', '"."'))
+            table = params['tableName']
+            sql = DremioPreviewClient.SQL_STATEMENT.format(schema=schema,
+                                                           table=table)
+
+            client = flight.FlightClient(self.url, **self.connection_args)
+            client.authenticate(_DremioAuthHandler(self.username, self.password))
+            flight_descriptor = flight.FlightDescriptor.for_command(sql)
+            flight_info = client.get_flight_info(flight_descriptor)
+            reader = client.do_get(flight_info.endpoints[0].ticket)
+
+            result = reader.read_all()
+            names = result.schema.names
+            types = result.schema.types
+
+            columns = map(lambda x: x.to_pylist(), result.columns)
+            rows = [dict(zip(names, row)) for row in zip(*columns)]
+            column_items = [ColumnItem(n, t) for n, t in zip(names, types)]
+
+            preview_data = PreviewData(column_items, rows)
+
+            data = PreviewDataSchema().dump(preview_data)[0]
+            errors = PreviewDataSchema().load(data)[1]
+            if errors:
+                logging.error(f'Error(s) occurred while building preview data: {errors}')
+                payload = jsonify({'preview_data': {}})
+                return make_response(payload, HTTPStatus.INTERNAL_SERVER_ERROR)
+            else:
+                payload = jsonify({'preview_data': data})
+                return make_response(payload, HTTPStatus.OK)
+
+        except Exception as e:
+            logging.error(f'Encountered exception: {e}')
+            payload = jsonify({'preview_data': {}})
+            return make_response(payload, HTTPStatus.INTERNAL_SERVER_ERROR)

amundsen_application/config.py

@@ -68,6 +68,10 @@ class Config:
     PREVIEW_CLIENT_ENABLED = os.getenv('PREVIEW_CLIENT_ENABLED') == 'true'  # type: bool
     # Maps to a class path and name
     PREVIEW_CLIENT = os.getenv('PREVIEW_CLIENT', None)  # type: Optional[str]
+    PREVIEW_CLIENT_URL = os.getenv('PREVIEW_CLIENT_URL')  # type: Optional[str]
+    PREVIEW_CLIENT_USERNAME = os.getenv('PREVIEW_CLIENT_USERNAME')  # type: Optional[str]
+    PREVIEW_CLIENT_PASSWORD = os.getenv('PREVIEW_CLIENT_PASSWORD')  # type: Optional[str]
+    PREVIEW_CLIENT_CERTIFICATE = os.getenv('PREVIEW_CLIENT_CERTIFICATE')  # type: Optional[str]
 
     # Settings for Announcement Client integration
     ANNOUNCEMENT_CLIENT_ENABLED = os.getenv('ANNOUNCEMENT_CLIENT_ENABLED') == 'true'  # type: bool

amundsen_application/static/.betterer.results

@@ -33,8 +33,8 @@ exports[`eslint`] = {
       [33, 6, 25, "Use object destructuring.", "354229464"],
       [34, 6, 29, "Use object destructuring.", "2645724888"]
     ],
-    "js/components/AvatarLabel/index.tsx:977373330": [
-      [16, 6, 39, "Variable name \`AvatarLabel\` must match one of the following formats: camelCase, UPPER_CASE", "3862143033"]
+    "js/components/AvatarLabel/index.tsx:1662930605": [
+      [17, 6, 39, "Variable name \`AvatarLabel\` must match one of the following formats: camelCase, UPPER_CASE", "3862143033"]
     ],
     "js/components/BadgeList/index.spec.tsx:3460985082": [
       [18, 4, 10, "Property name \`badge_name\` must match one of the following formats: camelCase", "1663829240"],
@@ -693,13 +693,13 @@ exports[`eslint`] = {
     "js/components/Tags/index.tsx:3468508233": [
       [38, 4, 21, "Must use destructuring props assignment", "4236634811"]
     ],
-    "js/config/config-default.ts:1841535925": [
+    "js/config/config-default.ts:1262490958": [
       [2, 0, 72, "\`../interfaces\` import should occur before import of \`./config-types\`", "1449508543"],
       [46, 6, 10, "Property name \`use_router\` must match one of the following formats: camelCase", "2026345778"],
       [52, 6, 10, "Property name \`use_router\` must match one of the following formats: camelCase", "2026345778"],
-      [168, 8, 10, "Property name \`sort_order\` must match one of the following formats: camelCase", "1697854094"],
-      [210, 6, 21, "\'partitionKey\' is defined but never used.", "399589312"],
-      [211, 6, 23, "\'partitionValue\' is defined but never used.", "793372348"]
+      [172, 8, 10, "Property name \`sort_order\` must match one of the following formats: camelCase", "1697854094"],
+      [218, 6, 21, "\'partitionKey\' is defined but never used.", "399589312"],
+      [219, 6, 23, "\'partitionValue\' is defined but never used.", "793372348"]
     ],
     "js/config/config-types.ts:3768974062": [
       [156, 2, 6, "Enum Member name \`DANGER\` must match one of the following formats: camelCase", "2553023038"],
@@ -1960,7 +1960,7 @@ exports[`eslint`] = {
       [22, 2, 17, "Property name \`recent_view_count\` must match one of the following formats: camelCase", "3048879712"],
       [25, 2, 17, "Property name \`updated_timestamp\` must match one of the following formats: camelCase", "1885086497"]
     ],
-    "js/interfaces/Enums.ts:2496623588": [
+    "js/interfaces/Enums.ts:2000518745": [
       [1, 2, 3, "Enum Member name \`PUT\` must match one of the following formats: camelCase", "193465492"],
       [2, 2, 6, "Enum Member name \`DELETE\` must match one of the following formats: camelCase", "2557553692"],
       [6, 2, 15, "Enum Member name \`CHECKBOX_SELECT\` must match one of the following formats: camelCase", "3914631617"],
@@ -2004,13 +2004,14 @@ exports[`eslint`] = {
       [55, 2, 8, "Enum Member name \`DATABASE\` must match one of the following formats: camelCase", "1787457952"],
       [56, 2, 4, "Enum Member name \`HIVE\` must match one of the following formats: camelCase", "2089119255"],
       [57, 2, 8, "Enum Member name \`BIGQUERY\` must match one of the following formats: camelCase", "2775320131"],
-      [58, 2, 5, "Enum Member name \`DRUID\` must match one of the following formats: camelCase", "208312203"],
-      [59, 2, 6, "Enum Member name \`PRESTO\` must match one of the following formats: camelCase", "3324391722"],
-      [60, 2, 8, "Enum Member name \`POSTGRES\` must match one of the following formats: camelCase", "2048417342"],
-      [61, 2, 8, "Enum Member name \`REDSHIFT\` must match one of the following formats: camelCase", "3620745238"],
-      [62, 2, 9, "Enum Member name \`SNOWFLAKE\` must match one of the following formats: camelCase", "2253633893"],
-      [67, 2, 7, "Enum Member name \`REGULAR\` must match one of the following formats: camelCase", "112855263"],
-      [68, 2, 5, "Enum Member name \`SMALL\` must match one of the following formats: camelCase", "232618394"]
+      [58, 2, 6, "Enum Member name \`DREMIO\` must match one of the following formats: camelCase", "2579905821"],
+      [59, 2, 5, "Enum Member name \`DRUID\` must match one of the following formats: camelCase", "208312203"],
+      [60, 2, 6, "Enum Member name \`PRESTO\` must match one of the following formats: camelCase", "3324391722"],
+      [61, 2, 8, "Enum Member name \`POSTGRES\` must match one of the following formats: camelCase", "2048417342"],
+      [62, 2, 8, "Enum Member name \`REDSHIFT\` must match one of the following formats: camelCase", "3620745238"],
+      [63, 2, 9, "Enum Member name \`SNOWFLAKE\` must match one of the following formats: camelCase", "2253633893"],
+      [68, 2, 7, "Enum Member name \`REGULAR\` must match one of the following formats: camelCase", "112855263"],
+      [69, 2, 5, "Enum Member name \`SMALL\` must match one of the following formats: camelCase", "232618394"]
     ],
     "js/interfaces/Feedback.ts:175362719": [
       [1, 2, 5, "Enum Member name \`ERROR\` must match one of the following formats: camelCase", "202381725"],
@@ -2479,7 +2480,7 @@ exports[`eslint`] = {
       [19, 55, 10, "Prop spreading is forbidden", "480399587"],
       [95, 10, 11, "Property name \`source_type\` must match one of the following formats: camelCase", "2336271487"]
     ],
-    "js/pages/TableDetailPage/SourceLink/index.tsx:1337415295": [
+    "js/pages/TableDetailPage/SourceLink/index.tsx:2340651505": [
       [18, 6, 37, "Variable name \`SourceLink\` must match one of the following formats: camelCase, UPPER_CASE", "1610242585"]
     ],
     "js/pages/TableDetailPage/TableHeaderBullets/index.spec.tsx:3978961134": [

amundsen_application/static/css/_icons.scss

@@ -15,6 +15,7 @@ $data-stores: (
   hive: '../images/icons/logo-hive.svg',
   bigquery: '../images/icons/logo-bigquery.svg',
   delta: '../images/icons/logo-delta.png',
+  dremio: '../images/icons/logo-dremio.svg',
   druid: '../images/icons/logo-druid.svg',
   presto: '../images/icons/logo-presto.svg',
   postgres: '../images/icons/logo-postgres.svg',

amundsen_application/static/images/icons/logo-dremio.svg

+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 42 42"><g fill-rule="evenodd"><path d="m16.81 22.51c-.83 2.17-3.25 3.2-3.92 2.83-.34-.18 0-.87 0-1.34-.49.19-1.3.55-1.79.75a5 5 0 0 1 0 7.59c2 3 1 5.58-1.79 7.59l-.11.07-.06-.2a9.45 9.45 0 0 0 -1.92-3.77c-.49-.57-1.05-1.09-1.6-1.61-.72-.68-1.42-1.38-2.07-2.12-2.75-2.88-3.84-7.95-3.48-11.87a15.28 15.28 0 0 1 1.74-5.75 18.11 18.11 0 0 1 2.8-3.91 20.67 20.67 0 0 1 9.07-5.67 12.63 12.63 0 0 1 9.32.43 10.45 10.45 0 0 1 1.54 1l.71.67.08.08 13.37-6.85c1.1-.55 1.93-.55 2.3-.15s-.15 1.43-.93 2l-11.5 8.31a2.61 2.61 0 0 1 -.64 2.2c1.53.53.18 2.77-.49 3.57-1.92 2.27-5.43 4.05-8.56 5.35-.84.36-1.32.51-2.07.79m-7.48 3c-1.25.57-3.69 1.84-3.46 3.22.32.85 1.31 0 1.68-.4a7.24 7.24 0 0 0 1.78-2.77z" fill="#383c42"/><path d="m27.3 14.66c.86-1.08.26-1.7-.34-1.1a5.24 5.24 0 0 1 -4.57 1.36c-2.35-.39-2.46-.75-2.06-1.1a6.39 6.39 0 0 0 7.34-2.49c.58-.83-1.36-1.49-2.5-3-5.48-7.44-22.64-.99-24.28 11.94a14.46 14.46 0 0 0 3.27 11.23c2 2.27 4.18 3.22 5.41 6.79 2.4-2.06 2-4.39.27-6.43a4 4 0 0 0 .46-6.86c-1 3.61-4.51 6.28-5.33 3.72-.53-2.61 3.77-4.34 10.65-7 5.76-2.16 9.48-3.94 11.68-7.12" fill="#31d3db"/><path d="m27.3 14.66c.86-1.08.26-1.7-.34-1.1a5.18 5.18 0 0 1 -3.96 1.44c-2.37-.17-4.26-.64-6-.45a15.16 15.16 0 0 0 -8.52 4.32c-3 3.08-6.32 9.48-3 10.72a1.49 1.49 0 0 1 -.48-.86c-.53-2.61 3.77-4.34 10.65-7 5.76-2.16 9.48-3.94 11.68-7.12" fill="#fff"/><path d="m12.55 19.71c1.77.87 1.42 3.91 1.16 4.67 3.36-1 2.36-5.93 3.92-7.45-2-.06-5.62 1.91-5.08 2.78" fill="#31d3db"/><path d="m18.23 10.69a1 1 0 1 1 .92 1.72 1 1 0 0 1 -.92-1.72" fill="#383c42"/><path d="m26.39 11.1 13.49-9.39c.38-.27.88-.85.67-1.13s-.79-.13-1.29.1l-14.58 7.88c-.47 1 .48 3.05 1.72 2.54" fill="#31d3db"/><path d="m25.21 9.25 14.09-8.25a2.31 2.31 0 0 1 1.23-.4c-.21-.24-.79-.1-1.27.13l-14.58 7.83a2.22 2.22 0 0 0 .89 2.51 3.16 3.16 0 0 1 -.36-1.83" fill="#fff"/><path d="m4.45 12.93a.67.67 0 0 1 .81.71 1 1 0 0 1 -.84 1 .67.67 0 0 1 -.8-.7 1.06 1.06 0 0 1 .83-1" fill="#21aaad"/><path d="m7.18 9.92a.59.59 0 0 1 .71.63 1 1 0 0 1 -.74.89.59.59 0 0 1 -.71-.63 1 1 0 0 1 .74-.89" fill="#21aaad"/><path d="m15.46 5.93a.59.59 0 0 1 .54.77.94.94 0 0 1 -.91.69.6.6 0 0 1 -.56-.77.94.94 0 0 1 .92-.69" fill="#21aaad"/><path d="m18 5.52a.5.5 0 0 1 .37.71.8.8 0 0 1 -.85.45.5.5 0 0 1 -.37-.68.8.8 0 0 1 .85-.48" fill="#21aaad"/><path d="m7.21 11.81a.44.44 0 0 1 .53.47.68.68 0 0 1 -.55.65.44.44 0 0 1 -.53-.46.71.71 0 0 1 .55-.66" fill="#21aaad"/><path d="m2.95 15.35a.45.45 0 0 1 .55.48.72.72 0 0 1 -.57.68.47.47 0 0 1 -.56-.51.72.72 0 0 1 .58-.68" fill="#21aaad"/><path d="m38 1.34-.37.2a2 2 0 0 0 .67 1.13 3.32 3.32 0 0 1 -.3-1.33m-3.2 1.72-.42.23c0 .54.39 1.35.83 1.46a2.94 2.94 0 0 1 -.36-1.69zm-3.32 1.8-.55.3a2.54 2.54 0 0 0 1.13 1.84 4.36 4.36 0 0 1 -.53-2.14zm-3.31 1.82-.63.32a2.89 2.89 0 0 0 1.34 2.21 4.43 4.43 0 0 1 -.71-2.53z" fill="#21aaad"/></g></svg>

amundsen_application/static/js/components/AvatarLabel/index.tsx

@@ -11,25 +11,27 @@ export interface AvatarLabelProps {
   avatarClass?: string;
   labelClass?: string;
   label?: string;
+  round?: boolean;
   src?: string;
 }
 
 const AvatarLabel: React.FC<AvatarLabelProps> = ({
   avatarClass,
-  labelClass,
-  label,
-  src,
+  labelClass = 'text-secondary',
+  label = '',
+  round = true,
+  src = '',
 }: AvatarLabelProps) => (
   <div className="avatar-label-component">
-    <Avatar className={avatarClass} name={label} src={src} size={24} round />
+    <Avatar
+      className={avatarClass}
+      name={label}
+      src={src}
+      size={24}
+      round={round}
+    />
     <span className={`avatar-label body-2 ${labelClass}`}>{label}</span>
   </div>
 );
 
-AvatarLabel.defaultProps = {
-  labelClass: 'text-secondary',
-  label: '',
-  src: '',
-};
-
 export default AvatarLabel;

amundsen_application/static/js/config/config-default.ts

@@ -108,6 +108,10 @@ const configDefault: AppConfig = {
           displayName: 'Delta',
           iconClass: 'icon-delta',
         },
+        dremio: {
+          displayName: 'Dremio',
+          iconClass: 'icon-dremio',
+        },
         druid: {
           displayName: 'Druid',
           iconClass: 'icon-druid',
@@ -178,6 +182,10 @@ const configDefault: AppConfig = {
         },
       },
       supportedDescriptionSources: {
+        dremio: {
+          displayName: 'Dremio',
+          iconPath: '/static/images/icons/logo-dremio.svg',
+        },
         github: {
           displayName: 'Github',
           iconPath: '/static/images/github.png',

amundsen_application/static/js/interfaces/Enums.ts

@@ -56,6 +56,7 @@ export enum IconType {
   DATABASE = 'icon-database',
   HIVE = 'icon-hive',
   BIGQUERY = 'icon-bigquery',
+  DREMIO = 'icon-dremio',
   DRUID = 'icon-druid',
   PRESTO = 'icon-presto',
   POSTGRES = 'icon-postgres',

amundsen_application/static/js/pages/TableDetailPage/SourceLink/index.tsx

@@ -33,6 +33,7 @@ const SourceLink: React.FC<SourceLinkProps> = ({
       <AvatarLabel
         label={getDescriptionSourceDisplayName(tableSource.source_type)}
         src={getDescriptionSourceIconPath(tableSource.source_type)}
+        round={false}
       />
     </a>
   );

amundsen_application/static/package-lock.json

@@ -22366,7 +22366,8 @@
       "version": "1.3.0",
       "resolved": "https://registry.npmjs.org/growly/-/growly-1.3.0.tgz",
       "integrity": "sha1-8QdIy+dq+WS3yWyTxrzCivEgwIE=",
-      "dev": true
+      "dev": true,
+      "optional": true
     },
     "gud": {
       "version": "1.0.0",
@@ -29863,6 +29864,7 @@
       "resolved": "https://registry.npmjs.org/node-notifier/-/node-notifier-8.0.1.tgz",
       "integrity": "sha512-BvEXF+UmsnAfYfoapKM9nGxnP+Wn7P91YfXmrKnfcYCx6VBeoN5Ez5Ogck6I8Bi5k4RlpqRYaw75pAwzX9OphA==",
       "dev": true,
+      "optional": true,
       "requires": {
         "growly": "^1.3.0",
         "is-wsl": "^2.2.0",
@@ -29877,6 +29879,7 @@
           "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
           "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
           "dev": true,
+          "optional": true,
           "requires": {
             "yallist": "^4.0.0"
           }
@@ -29886,6 +29889,7 @@
           "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.4.tgz",
           "integrity": "sha512-tCfb2WLjqFAtXn4KEdxIhalnRtoKFN7nAwj0B3ZXCbQloV2tq5eDbcTmT68JJD3nRJq24/XgxtQKFIpQdtvmVw==",
           "dev": true,
+          "optional": true,
           "requires": {
             "lru-cache": "^6.0.0"
           }
@@ -29895,6 +29899,7 @@
           "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
           "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==",
           "dev": true,
+          "optional": true,
           "requires": {
             "isexe": "^2.0.0"
           }
@@ -35056,7 +35061,8 @@
       "version": "0.1.1",
       "resolved": "https://registry.npmjs.org/shellwords/-/shellwords-0.1.1.tgz",
       "integrity": "sha512-vFwSUfQvqybiICwZY5+DAWIPLKsWO31Q91JSKl3UYv+K5c2QRPzn0qzec6QPu1Qc9eHYItiP3NdJqNVqetYAww==",
-      "dev": true
+      "dev": true,
+      "optional": true
     },
     "side-channel": {
       "version": "1.0.2",

requirements.txt

@@ -71,6 +71,9 @@ responses==0.12.1
 # Required for announcement client
 SQLAlchemy==1.3.22
 
+# Required for Dremio preview client
+pyarrow==2.0.0
+
 # A common package that holds the models deifnition and schemas that are used
 # accross different amundsen repositories.
 amundsen-common==0.5.9