Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
F
Flask_databasemigration
Project
Project
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Masarath Fatima
Flask_databasemigration
Commits
2dc5b71f
Commit
2dc5b71f
authored
Jan 27, 2022
by
Masarath Fatima
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Delete postgrespool.py
parent
833d33de
Changes
1
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
with
0 additions
and
86 deletions
+0
-86
postgrespool.py
flaskdatabasemigrations/onpremisesdatabases/postgrespool.py
+0
-86
No files found.
flaskdatabasemigrations/onpremisesdatabases/postgrespool.py
deleted
100644 → 0
View file @
833d33de
import
configparser
import
os
import
pickle
import
pandas
as
pd
from
flask
import
request
from
google.cloud
import
bigquery
from
google.cloud.exceptions
import
NotFound
from
onpremisesdatabases.postgres1
import
POSTGRESPOOL
class
PQ
:
def
POSTGRESPOOl
(
self
):
config
=
configparser
.
ConfigParser
()
select
=
request
.
form
.
get
(
'comp_select'
)
global
list1
list1
=
[]
if
select
==
"POSTGRES"
:
postgrespool
=
POSTGRESPOOL
()
try
:
config
.
read
(
"C:
\\
Users
\\
mfatima
\\
PycharmProjects
\\
flaskdatabasemigrations
\\
configure
\\
db_config.ini"
)
except
FileNotFoundError
as
e
:
print
(
e
)
databases
=
config
[
'DATABASES'
][
'postgres_db_list'
]
db_lists
=
databases
.
split
(
","
)
# ['db1',"db2"]
for
db
in
db_lists
:
try
:
config
.
read
(
f
"C:
\\
Users
\\
mfatima
\\
PycharmProjects
\\
flaskdatabasemigrations
\\
configure
\\
{db}_config.ini"
)
except
FileNotFoundError
as
e
:
\
print
(
e
)
try
:
schema_df
=
pd
.
read_csv
(
f
"C:
\\
Users
\\
mfatima
\\
PycharmProjects
\\
flaskdatabasemigrations
\\
schemafiles
\\
{db}_config.csv"
)
except
FileNotFoundError
as
e
:
print
(
e
)
db_user
=
config
[
'Server_Credentials'
][
'db_user'
]
db_pass
=
config
[
'Server_Credentials'
][
'db_password'
]
db_server
=
config
[
'Server_Credentials'
][
'db_host'
]
db_name
=
config
[
'Database'
][
'db_name'
]
table_name
=
config
[
'Database'
][
'table_names'
]
table_names_list
=
table_name
.
split
(
","
)
print
(
table_names_list
)
conn
=
postgrespool
.
get_connection
(
db_server
,
db_user
,
db_pass
)
cursor
=
conn
.
cursor
()
len_table
=
len
(
table_names_list
)
# list1 = []
for
table
in
table_names_list
:
print
(
table
)
df
=
postgrespool
.
read_and_prepare_data
(
db_name
,
table
,
conn
)
project_id
=
config
[
'Google'
][
'PROJECT_ID'
]
dataset_id
=
config
[
'Google'
][
'DATASET_ID'
]
os
.
environ
[
"GOOGLE_APPLICATION_CREDENTIALS"
]
=
config
[
'Google'
][
'CREDENTIALS'
]
get_unique_columns
=
pd
.
unique
(
schema_df
[
'table'
])
demo_df
=
schema_df
[
schema_df
[
'table'
]
==
table
]
schema_df1
=
demo_df
[[
'name'
,
'type'
,
'mode'
]]
schema_json
=
schema_df1
.
to_dict
(
'records'
)
table_id
=
'{0}.{1}'
.
format
(
dataset_id
,
table
)
postgrespool
.
load_into_bq
(
df
,
table_id
,
project_id
,
schema_json
)
print
(
f
"loaded successfully {table}"
)
# status=True
client
=
bigquery
.
Client
()
try
:
client
.
get_table
(
table_id
)
status
=
True
except
NotFound
:
status
=
False
list
=
[
select
,
db_name
,
table
,
status
]
list1
.
append
(
list
)
print
(
list1
)
filename
=
'my_data'
outfile
=
open
(
filename
,
'wb'
)
pickle
.
dump
(
list1
,
outfile
)
outfile
.
close
()
else
:
print
(
"Selected None"
);
def
alist
(
self
):
global
list1
return
list1
\ No newline at end of file
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment