Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
A
AmendsenProject
Project
Project
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Shaik Janipasha
AmendsenProject
Commits
0339dd55
Unverified
Commit
0339dd55
authored
Aug 26, 2020
by
Alagappan
Committed by
GitHub
Aug 26, 2020
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
chore: Sample dags minor cleanup (#344)
Signed-off-by:
Tao Feng
<
fengtao04@gmail.com
>
parent
d9f7d9dd
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
10 additions
and
6 deletions
+10
-6
hive_sample_dag.py
example/dags/hive_sample_dag.py
+4
-0
postgres_sample_dag.py
example/dags/postgres_sample_dag.py
+6
-6
No files found.
example/dags/hive_sample_dag.py
View file @
0339dd55
...
@@ -199,3 +199,7 @@ with DAG('amundsen_databuilder', default_args=default_args, **dag_args) as dag:
...
@@ -199,3 +199,7 @@ with DAG('amundsen_databuilder', default_args=default_args, **dag_args) as dag:
'watermark_type'
:
'"low_watermark"'
,
'watermark_type'
:
'"low_watermark"'
,
'part_regex'
:
'{}'
.
format
(
'{{ ds }}'
)}
'part_regex'
:
'{}'
.
format
(
'{{ ds }}'
)}
)
)
# Schedule high and low watermark task after metadata task
amundsen_databuilder_table_metadata_job
>>
amundsen_hwm_job
amundsen_databuilder_table_metadata_job
>>
amundsen_lwm_job
example/dags/postgres_sample_dag.py
View file @
0339dd55
...
@@ -75,7 +75,7 @@ def connection_string():
...
@@ -75,7 +75,7 @@ def connection_string():
return
"postgresql://
%
s:
%
s@
%
s:
%
s/
%
s"
%
(
user
,
password
,
host
,
port
,
db
)
return
"postgresql://
%
s:
%
s@
%
s:
%
s/
%
s"
%
(
user
,
password
,
host
,
port
,
db
)
def
create_table_extract_job
(
**
kwargs
):
def
create_table_extract_job
():
where_clause_suffix
=
textwrap
.
dedent
(
"""
where_clause_suffix
=
textwrap
.
dedent
(
"""
where table_schema in {schemas}
where table_schema in {schemas}
"""
)
.
format
(
schemas
=
SUPPORTED_SCHEMA_SQL_IN_CLAUSE
)
"""
)
.
format
(
schemas
=
SUPPORTED_SCHEMA_SQL_IN_CLAUSE
)
...
@@ -161,15 +161,15 @@ def create_es_publisher_sample_job():
...
@@ -161,15 +161,15 @@ def create_es_publisher_sample_job():
with
DAG
(
'amundsen_databuilder'
,
default_args
=
default_args
,
**
dag_args
)
as
dag
:
with
DAG
(
'amundsen_databuilder'
,
default_args
=
default_args
,
**
dag_args
)
as
dag
:
create
_table_extract_job
=
PythonOperator
(
postgres
_table_extract_job
=
PythonOperator
(
task_id
=
'
create
_table_extract_job'
,
task_id
=
'
postgres
_table_extract_job'
,
python_callable
=
create_table_extract_job
python_callable
=
create_table_extract_job
)
)
create
_es_index_job
=
PythonOperator
(
postgres
_es_index_job
=
PythonOperator
(
task_id
=
'
create
_es_publisher_sample_job'
,
task_id
=
'
postgres
_es_publisher_sample_job'
,
python_callable
=
create_es_publisher_sample_job
python_callable
=
create_es_publisher_sample_job
)
)
# elastic search update run after table metadata has been updated
# elastic search update run after table metadata has been updated
create_table_extract_job
>>
create
_es_index_job
postgres_table_extract_job
>>
postgres
_es_index_job
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment