Skip to content

A
AmendsenProject
Unverified Commit e3b83d32 authored by Josh Howard's avatar Josh Howard Committed by GitHub
Browse files
Options

feat: Integration with Dremio (#859) 

* Added Dremio preview example
Signed-off-by: 's avatarJosh Howard <joshthoward@gmail.com>

* Checkpoint on flight preview
Signed-off-by: 's avatarJosh Howard <joshthoward@gmail.com>

* Reworked flight preview to construct new client on each invocation; somehow reduces loading times
Signed-off-by: 's avatarJosh Howard <joshthoward@gmail.com>

* Updated table icon and sources link to include Dremio
Signed-off-by: 's avatarJosh Howard <joshthoward@gmail.com>

* Fixed flake errors
Signed-off-by: 's avatarJosh Howard <joshthoward@gmail.com>

* Fixed code review comments
Signed-off-by: 's avatarJosh Howard <joshthoward@gmail.com>
parent c8fa032d
Hide whitespace changes
Inline Side-by-side
Showing with 151 additions and 27 deletions
amundsen_application/base/examples/example_dremio_preview_client.py 0 → 100644
View file @ e3b83d32
# Copyright Contributors to the Amundsen project.
# SPDX-License-Identifier: Apache-2.0
from http import HTTPStatus
import logging
from typing import Dict # noqa: F401
from flask import Response, jsonify, make_response, current_app as app
from pyarrow import flight
from amundsen_application.base.base_superset_preview_client import BasePreviewClient
from amundsen_application.models.preview_data import PreviewData, PreviewDataSchema, ColumnItem
class _DremioAuthHandler(flight.ClientAuthHandler):
"""ClientAuthHandler for connections to Dremio server endpoint.
"""
def __init__(self, username: str, password: str) -> None:
self.username = username
self.password = password
super(flight.ClientAuthHandler, self).__init__()
def authenticate(self, outgoing: flight.ClientAuthSender,
incoming: flight.ClientAuthReader) -> None:
"""Authenticate with Dremio user credentials.
"""
basic_auth = flight.BasicAuth(self.username, self.password)
outgoing.write(basic_auth.serialize())
self.token = incoming.read()
def get_token(self,) -> str:
"""Get the token from this AuthHandler.
"""
return self.token
class DremioPreviewClient(BasePreviewClient):
SQL_STATEMENT = 'SELECT * FROM {schema}."{table}" LIMIT 50'
def __init__(self,) -> None:
self.url = app.config['PREVIEW_CLIENT_URL']
self.username = app.config['PREVIEW_CLIENT_USERNAME']
self.password = app.config['PREVIEW_CLIENT_PASSWORD']
self.connection_args: Dict[str, bytes] = {}
tls_root_certs_path = app.config['PREVIEW_CLIENT_CERTIFICATE']
if tls_root_certs_path is not None:
with open(tls_root_certs_path, "rb") as f:
self.connection_args["tls_root_certs"] = f.read()
def get_preview_data(self, params: Dict, optionalHeaders: Dict = None) -> Response:
"""Preview data from Dremio source
"""
database = params.get('database')
if database != 'DREMIO':
logging.info('Skipping table preview for non-Dremio table')
return make_response(jsonify({'preview_data': {}}), HTTPStatus.OK)
try:
# Format base SQL_STATEMENT with request table and schema
schema = '"{}"'.format(params['schema'].replace('.', '"."'))
table = params['tableName']
sql = DremioPreviewClient.SQL_STATEMENT.format(schema=schema,
table=table)
client = flight.FlightClient(self.url, **self.connection_args)
client.authenticate(_DremioAuthHandler(self.username, self.password))
flight_descriptor = flight.FlightDescriptor.for_command(sql)
flight_info = client.get_flight_info(flight_descriptor)
reader = client.do_get(flight_info.endpoints[0].ticket)
result = reader.read_all()
names = result.schema.names
types = result.schema.types
columns = map(lambda x: x.to_pylist(), result.columns)
rows = [dict(zip(names, row)) for row in zip(*columns)]
column_items = [ColumnItem(n, t) for n, t in zip(names, types)]
preview_data = PreviewData(column_items, rows)
data = PreviewDataSchema().dump(preview_data)[0]
errors = PreviewDataSchema().load(data)[1]
if errors:
logging.error(f'Error(s) occurred while building preview data: {errors}')
payload = jsonify({'preview_data': {}})
return make_response(payload, HTTPStatus.INTERNAL_SERVER_ERROR)
else:
payload = jsonify({'preview_data': data})
return make_response(payload, HTTPStatus.OK)
except Exception as e:
logging.error(f'Encountered exception: {e}')
payload = jsonify({'preview_data': {}})
return make_response(payload, HTTPStatus.INTERNAL_SERVER_ERROR)
amundsen_application/config.py
View file @ e3b83d32
......@@ -68,6 +68,10 @@ class Config:
PREVIEW_CLIENT_ENABLED = os.getenv('PREVIEW_CLIENT_ENABLED') == 'true' # type: bool
# Maps to a class path and name
PREVIEW_CLIENT = os.getenv('PREVIEW_CLIENT', None) # type: Optional[str]
PREVIEW_CLIENT_URL = os.getenv('PREVIEW_CLIENT_URL') # type: Optional[str]
PREVIEW_CLIENT_USERNAME = os.getenv('PREVIEW_CLIENT_USERNAME') # type: Optional[str]
PREVIEW_CLIENT_PASSWORD = os.getenv('PREVIEW_CLIENT_PASSWORD') # type: Optional[str]
PREVIEW_CLIENT_CERTIFICATE = os.getenv('PREVIEW_CLIENT_CERTIFICATE') # type: Optional[str]
# Settings for Announcement Client integration
ANNOUNCEMENT_CLIENT_ENABLED = os.getenv('ANNOUNCEMENT_CLIENT_ENABLED') == 'true' # type: bool
......
amundsen_application/static/.betterer.results
View file @ e3b83d32
......@@ -33,8 +33,8 @@ exports[`eslint`] = {
[33, 6, 25, "Use object destructuring.", "354229464"],
[34, 6, 29, "Use object destructuring.", "2645724888"]
],
"js/components/AvatarLabel/index.tsx:977373330": [
[16, 6, 39, "Variable name \`AvatarLabel\` must match one of the following formats: camelCase, UPPER_CASE", "3862143033"]
"js/components/AvatarLabel/index.tsx:1662930605": [
[17, 6, 39, "Variable name \`AvatarLabel\` must match one of the following formats: camelCase, UPPER_CASE", "3862143033"]
],
"js/components/BadgeList/index.spec.tsx:3460985082": [
[18, 4, 10, "Property name \`badge_name\` must match one of the following formats: camelCase", "1663829240"],
......@@ -693,13 +693,13 @@ exports[`eslint`] = {
"js/components/Tags/index.tsx:3468508233": [
[38, 4, 21, "Must use destructuring props assignment", "4236634811"]
],
"js/config/config-default.ts:1841535925": [
"js/config/config-default.ts:1262490958": [
[2, 0, 72, "\`../interfaces\` import should occur before import of \`./config-types\`", "1449508543"],
[46, 6, 10, "Property name \`use_router\` must match one of the following formats: camelCase", "2026345778"],
[52, 6, 10, "Property name \`use_router\` must match one of the following formats: camelCase", "2026345778"],
[168, 8, 10, "Property name \`sort_order\` must match one of the following formats: camelCase", "1697854094"],
[210, 6, 21, "\'partitionKey\' is defined but never used.", "399589312"],
[211, 6, 23, "\'partitionValue\' is defined but never used.", "793372348"]
[172, 8, 10, "Property name \`sort_order\` must match one of the following formats: camelCase", "1697854094"],
[218, 6, 21, "\'partitionKey\' is defined but never used.", "399589312"],
[219, 6, 23, "\'partitionValue\' is defined but never used.", "793372348"]
],
"js/config/config-types.ts:3768974062": [
[156, 2, 6, "Enum Member name \`DANGER\` must match one of the following formats: camelCase", "2553023038"],
......@@ -1960,7 +1960,7 @@ exports[`eslint`] = {
[22, 2, 17, "Property name \`recent_view_count\` must match one of the following formats: camelCase", "3048879712"],
[25, 2, 17, "Property name \`updated_timestamp\` must match one of the following formats: camelCase", "1885086497"]
],
"js/interfaces/Enums.ts:2496623588": [
"js/interfaces/Enums.ts:2000518745": [
[1, 2, 3, "Enum Member name \`PUT\` must match one of the following formats: camelCase", "193465492"],
[2, 2, 6, "Enum Member name \`DELETE\` must match one of the following formats: camelCase", "2557553692"],
[6, 2, 15, "Enum Member name \`CHECKBOX_SELECT\` must match one of the following formats: camelCase", "3914631617"],
......@@ -2004,13 +2004,14 @@ exports[`eslint`] = {
[55, 2, 8, "Enum Member name \`DATABASE\` must match one of the following formats: camelCase", "1787457952"],
[56, 2, 4, "Enum Member name \`HIVE\` must match one of the following formats: camelCase", "2089119255"],
[57, 2, 8, "Enum Member name \`BIGQUERY\` must match one of the following formats: camelCase", "2775320131"],
[58, 2, 5, "Enum Member name \`DRUID\` must match one of the following formats: camelCase", "208312203"],
[59, 2, 6, "Enum Member name \`PRESTO\` must match one of the following formats: camelCase", "3324391722"],
[60, 2, 8, "Enum Member name \`POSTGRES\` must match one of the following formats: camelCase", "2048417342"],
[61, 2, 8, "Enum Member name \`REDSHIFT\` must match one of the following formats: camelCase", "3620745238"],
[62, 2, 9, "Enum Member name \`SNOWFLAKE\` must match one of the following formats: camelCase", "2253633893"],
[67, 2, 7, "Enum Member name \`REGULAR\` must match one of the following formats: camelCase", "112855263"],
[68, 2, 5, "Enum Member name \`SMALL\` must match one of the following formats: camelCase", "232618394"]
[58, 2, 6, "Enum Member name \`DREMIO\` must match one of the following formats: camelCase", "2579905821"],
[59, 2, 5, "Enum Member name \`DRUID\` must match one of the following formats: camelCase", "208312203"],
[60, 2, 6, "Enum Member name \`PRESTO\` must match one of the following formats: camelCase", "3324391722"],
[61, 2, 8, "Enum Member name \`POSTGRES\` must match one of the following formats: camelCase", "2048417342"],
[62, 2, 8, "Enum Member name \`REDSHIFT\` must match one of the following formats: camelCase", "3620745238"],
[63, 2, 9, "Enum Member name \`SNOWFLAKE\` must match one of the following formats: camelCase", "2253633893"],
[68, 2, 7, "Enum Member name \`REGULAR\` must match one of the following formats: camelCase", "112855263"],
[69, 2, 5, "Enum Member name \`SMALL\` must match one of the following formats: camelCase", "232618394"]
],
"js/interfaces/Feedback.ts:175362719": [
[1, 2, 5, "Enum Member name \`ERROR\` must match one of the following formats: camelCase", "202381725"],
......@@ -2479,7 +2480,7 @@ exports[`eslint`] = {
[19, 55, 10, "Prop spreading is forbidden", "480399587"],
[95, 10, 11, "Property name \`source_type\` must match one of the following formats: camelCase", "2336271487"]
],
"js/pages/TableDetailPage/SourceLink/index.tsx:1337415295": [
"js/pages/TableDetailPage/SourceLink/index.tsx:2340651505": [
[18, 6, 37, "Variable name \`SourceLink\` must match one of the following formats: camelCase, UPPER_CASE", "1610242585"]
],
"js/pages/TableDetailPage/TableHeaderBullets/index.spec.tsx:3978961134": [
......
amundsen_application/static/css/_icons.scss
View file @ e3b83d32
......@@ -15,6 +15,7 @@ $data-stores: (
hive: '../images/icons/logo-hive.svg',
bigquery: '../images/icons/logo-bigquery.svg',
delta: '../images/icons/logo-delta.png',
dremio: '../images/icons/logo-dremio.svg',
druid: '../images/icons/logo-druid.svg',
presto: '../images/icons/logo-presto.svg',
postgres: '../images/icons/logo-postgres.svg',
......
amundsen_application/static/images/icons/logo-dremio.svg 0 → 100644
View file @ e3b83d32
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 42 42"><g fill-rule="evenodd"><path d="m16.81 22.51c-.83 2.17-3.25 3.2-3.92 2.83-.34-.18 0-.87 0-1.34-.49.19-1.3.55-1.79.75a5 5 0 0 1 0 7.59c2 3 1 5.58-1.79 7.59l-.11.07-.06-.2a9.45 9.45 0 0 0 -1.92-3.77c-.49-.57-1.05-1.09-1.6-1.61-.72-.68-1.42-1.38-2.07-2.12-2.75-2.88-3.84-7.95-3.48-11.87a15.28 15.28 0 0 1 1.74-5.75 18.11 18.11 0 0 1 2.8-3.91 20.67 20.67 0 0 1 9.07-5.67 12.63 12.63 0 0 1 9.32.43 10.45 10.45 0 0 1 1.54 1l.71.67.08.08 13.37-6.85c1.1-.55 1.93-.55 2.3-.15s-.15 1.43-.93 2l-11.5 8.31a2.61 2.61 0 0 1 -.64 2.2c1.53.53.18 2.77-.49 3.57-1.92 2.27-5.43 4.05-8.56 5.35-.84.36-1.32.51-2.07.79m-7.48 3c-1.25.57-3.69 1.84-3.46 3.22.32.85 1.31 0 1.68-.4a7.24 7.24 0 0 0 1.78-2.77z" fill="#383c42"/><path d="m27.3 14.66c.86-1.08.26-1.7-.34-1.1a5.24 5.24 0 0 1 -4.57 1.36c-2.35-.39-2.46-.75-2.06-1.1a6.39 6.39 0 0 0 7.34-2.49c.58-.83-1.36-1.49-2.5-3-5.48-7.44-22.64-.99-24.28 11.94a14.46 14.46 0 0 0 3.27 11.23c2 2.27 4.18 3.22 5.41 6.79 2.4-2.06 2-4.39.27-6.43a4 4 0 0 0 .46-6.86c-1 3.61-4.51 6.28-5.33 3.72-.53-2.61 3.77-4.34 10.65-7 5.76-2.16 9.48-3.94 11.68-7.12" fill="#31d3db"/><path d="m27.3 14.66c.86-1.08.26-1.7-.34-1.1a5.18 5.18 0 0 1 -3.96 1.44c-2.37-.17-4.26-.64-6-.45a15.16 15.16 0 0 0 -8.52 4.32c-3 3.08-6.32 9.48-3 10.72a1.49 1.49 0 0 1 -.48-.86c-.53-2.61 3.77-4.34 10.65-7 5.76-2.16 9.48-3.94 11.68-7.12" fill="#fff"/><path d="m12.55 19.71c1.77.87 1.42 3.91 1.16 4.67 3.36-1 2.36-5.93 3.92-7.45-2-.06-5.62 1.91-5.08 2.78" fill="#31d3db"/><path d="m18.23 10.69a1 1 0 1 1 .92 1.72 1 1 0 0 1 -.92-1.72" fill="#383c42"/><path d="m26.39 11.1 13.49-9.39c.38-.27.88-.85.67-1.13s-.79-.13-1.29.1l-14.58 7.88c-.47 1 .48 3.05 1.72 2.54" fill="#31d3db"/><path d="m25.21 9.25 14.09-8.25a2.31 2.31 0 0 1 1.23-.4c-.21-.24-.79-.1-1.27.13l-14.58 7.83a2.22 2.22 0 0 0 .89 2.51 3.16 3.16 0 0 1 -.36-1.83" fill="#fff"/><path d="m4.45 12.93a.67.67 0 0 1 .81.71 1 1 0 0 1 -.84 1 .67.67 0 0 1 -.8-.7 1.06 1.06 0 0 1 .83-1" fill="#21aaad"/><path d="m7.18 9.92a.59.59 0 0 1 .71.63 1 1 0 0 1 -.74.89.59.59 0 0 1 -.71-.63 1 1 0 0 1 .74-.89" fill="#21aaad"/><path d="m15.46 5.93a.59.59 0 0 1 .54.77.94.94 0 0 1 -.91.69.6.6 0 0 1 -.56-.77.94.94 0 0 1 .92-.69" fill="#21aaad"/><path d="m18 5.52a.5.5 0 0 1 .37.71.8.8 0 0 1 -.85.45.5.5 0 0 1 -.37-.68.8.8 0 0 1 .85-.48" fill="#21aaad"/><path d="m7.21 11.81a.44.44 0 0 1 .53.47.68.68 0 0 1 -.55.65.44.44 0 0 1 -.53-.46.71.71 0 0 1 .55-.66" fill="#21aaad"/><path d="m2.95 15.35a.45.45 0 0 1 .55.48.72.72 0 0 1 -.57.68.47.47 0 0 1 -.56-.51.72.72 0 0 1 .58-.68" fill="#21aaad"/><path d="m38 1.34-.37.2a2 2 0 0 0 .67 1.13 3.32 3.32 0 0 1 -.3-1.33m-3.2 1.72-.42.23c0 .54.39 1.35.83 1.46a2.94 2.94 0 0 1 -.36-1.69zm-3.32 1.8-.55.3a2.54 2.54 0 0 0 1.13 1.84 4.36 4.36 0 0 1 -.53-2.14zm-3.31 1.82-.63.32a2.89 2.89 0 0 0 1.34 2.21 4.43 4.43 0 0 1 -.71-2.53z" fill="#21aaad"/></g></svg>
amundsen_application/static/js/components/AvatarLabel/index.tsx
View file @ e3b83d32
......@@ -11,25 +11,27 @@ export interface AvatarLabelProps {
avatarClass?: string;
labelClass?: string;
label?: string;
round?: boolean;
src?: string;
}
const AvatarLabel: React.FC<AvatarLabelProps> = ({
avatarClass,
labelClass,
label,
src,
labelClass = 'text-secondary',
label = '',
round = true,
src = '',
}: AvatarLabelProps) => (
<div className="avatar-label-component">
<Avatar className={avatarClass} name={label} src={src} size={24} round />
<Avatar
className={avatarClass}
name={label}
src={src}
size={24}
round={round}
/>
<span className={`avatar-label body-2 ${labelClass}`}>{label}</span>
</div>
);
AvatarLabel.defaultProps = {
labelClass: 'text-secondary',
label: '',
src: '',
};
export default AvatarLabel;
amundsen_application/static/js/config/config-default.ts
View file @ e3b83d32
......@@ -108,6 +108,10 @@ const configDefault: AppConfig = {
displayName: 'Delta',
iconClass: 'icon-delta',
},
dremio: {
displayName: 'Dremio',
iconClass: 'icon-dremio',
},
druid: {
displayName: 'Druid',
iconClass: 'icon-druid',
......@@ -178,6 +182,10 @@ const configDefault: AppConfig = {
},
},
supportedDescriptionSources: {
dremio: {
displayName: 'Dremio',
iconPath: '/static/images/icons/logo-dremio.svg',
},
github: {
displayName: 'Github',
iconPath: '/static/images/github.png',
......
amundsen_application/static/js/interfaces/Enums.ts
View file @ e3b83d32
......@@ -56,6 +56,7 @@ export enum IconType {
DATABASE = 'icon-database',
HIVE = 'icon-hive',
BIGQUERY = 'icon-bigquery',
DREMIO = 'icon-dremio',
DRUID = 'icon-druid',
PRESTO = 'icon-presto',
POSTGRES = 'icon-postgres',
......
amundsen_application/static/js/pages/TableDetailPage/SourceLink/index.tsx
View file @ e3b83d32
......@@ -33,6 +33,7 @@ const SourceLink: React.FC<SourceLinkProps> = ({
<AvatarLabel
label={getDescriptionSourceDisplayName(tableSource.source_type)}
src={getDescriptionSourceIconPath(tableSource.source_type)}
round={false}
/>
</a>
);
......
amundsen_application/static/package-lock.json
View file @ e3b83d32
......@@ -22366,7 +22366,8 @@
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/growly/-/growly-1.3.0.tgz",
"integrity": "sha1-8QdIy+dq+WS3yWyTxrzCivEgwIE=",
"dev": true
"dev": true,
"optional": true
},
"gud": {
"version": "1.0.0",
......@@ -29863,6 +29864,7 @@
"resolved": "https://registry.npmjs.org/node-notifier/-/node-notifier-8.0.1.tgz",
"integrity": "sha512-BvEXF+UmsnAfYfoapKM9nGxnP+Wn7P91YfXmrKnfcYCx6VBeoN5Ez5Ogck6I8Bi5k4RlpqRYaw75pAwzX9OphA==",
"dev": true,
"optional": true,
"requires": {
"growly": "^1.3.0",
"is-wsl": "^2.2.0",
......@@ -29877,6 +29879,7 @@
"resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz",
"integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==",
"dev": true,
"optional": true,
"requires": {
"yallist": "^4.0.0"
}
......@@ -29886,6 +29889,7 @@
"resolved": "https://registry.npmjs.org/semver/-/semver-7.3.4.tgz",
"integrity": "sha512-tCfb2WLjqFAtXn4KEdxIhalnRtoKFN7nAwj0B3ZXCbQloV2tq5eDbcTmT68JJD3nRJq24/XgxtQKFIpQdtvmVw==",
"dev": true,
"optional": true,
"requires": {
"lru-cache": "^6.0.0"
}
......@@ -29895,6 +29899,7 @@
"resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
"integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==",
"dev": true,
"optional": true,
"requires": {
"isexe": "^2.0.0"
}
......@@ -35056,7 +35061,8 @@
"version": "0.1.1",
"resolved": "https://registry.npmjs.org/shellwords/-/shellwords-0.1.1.tgz",
"integrity": "sha512-vFwSUfQvqybiICwZY5+DAWIPLKsWO31Q91JSKl3UYv+K5c2QRPzn0qzec6QPu1Qc9eHYItiP3NdJqNVqetYAww==",
"dev": true
"dev": true,
"optional": true
},
"side-channel": {
"version": "1.0.2",
requirements.txt
View file @ e3b83d32
......@@ -71,6 +71,9 @@ responses==0.12.1
# Required for announcement client
SQLAlchemy==1.3.22
# Required for Dremio preview client
pyarrow==2.0.0
# A common package that holds the models deifnition and schemas that are used
# accross different amundsen repositories.
amundsen-common==0.5.9
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment