Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
S
SparkCSVConnector
Project
Project
Details
Activity
Releases
Cycle Analytics
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Issues
0
Issues
0
List
Board
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Charts
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Venkata Pandu Ranga Bujjam
SparkCSVConnector
Commits
143bc927
Commit
143bc927
authored
May 29, 2023
by
Venkata Pandu Ranga Bujjam
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
added loggers
parent
e3addc7e
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
16 additions
and
4 deletions
+16
-4
Consumer.java
src/main/java/com/nisum/spark/consumer/Consumer.java
+8
-2
Publisher.java
src/main/java/com/nisum/spark/publisher/Publisher.java
+8
-2
No files found.
src/main/java/com/nisum/spark/consumer/Consumer.java
View file @
143bc927
package
com
.
nisum
.
spark
.
consumer
;
package
com
.
nisum
.
spark
.
consumer
;
import
com.nisum.spark.dto.Product
;
import
com.nisum.spark.dto.Product
;
import
org.apache.spark.sql.*
;
import
org.apache.spark.sql.*
;
import
org.apache.spark.sql.streaming.StreamingQuery
;
import
org.apache.spark.sql.streaming.StreamingQuery
;
import
org.apache.spark.sql.types.StructType
;
import
org.apache.spark.sql.types.StructType
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
import
java.util.HashMap
;
import
java.util.HashMap
;
import
java.util.Map
;
import
java.util.Map
;
...
@@ -12,11 +15,12 @@ import java.util.concurrent.TimeoutException;
...
@@ -12,11 +15,12 @@ import java.util.concurrent.TimeoutException;
import
static
com
.
nisum
.
spark
.
util
.
Constants
.*;
import
static
com
.
nisum
.
spark
.
util
.
Constants
.*;
public
class
Consumer
{
public
class
Consumer
{
static
Logger
log
=
LoggerFactory
.
getLogger
(
Consumer
.
class
);
public
static
void
main
(
String
[]
args
)
throws
TimeoutException
{
public
static
void
main
(
String
[]
args
)
throws
TimeoutException
{
SparkSession
sparkSession
=
SparkSession
SparkSession
sparkSession
=
SparkSession
.
builder
().
master
(
"local[*]"
).
appName
(
"Spark Streaming Example 2"
).
getOrCreate
();
.
builder
().
master
(
"local[*]"
).
appName
(
"Spark Streaming Example 2"
).
getOrCreate
();
log
.
debug
(
"spark session is created"
);
Map
<
String
,
String
>
kafkaConfigMap
=
new
HashMap
<>();
Map
<
String
,
String
>
kafkaConfigMap
=
new
HashMap
<>();
kafkaConfigMap
.
put
(
"kafka.bootstrap.servers"
,
HOST
);
kafkaConfigMap
.
put
(
"kafka.bootstrap.servers"
,
HOST
);
kafkaConfigMap
.
put
(
"subscribe"
,
TOPIC
);
kafkaConfigMap
.
put
(
"subscribe"
,
TOPIC
);
...
@@ -29,7 +33,7 @@ public class Consumer {
...
@@ -29,7 +33,7 @@ public class Consumer {
.
format
(
"kafka"
)
.
format
(
"kafka"
)
.
options
(
kafkaConfigMap
)
.
options
(
kafkaConfigMap
)
.
load
();
.
load
();
log
.
debug
(
"kafka is loaded"
);
if
(
df
!=
null
)
{
if
(
df
!=
null
)
{
StructType
structType
=
Encoders
.
bean
(
Product
.
class
).
schema
();
StructType
structType
=
Encoders
.
bean
(
Product
.
class
).
schema
();
...
@@ -56,6 +60,7 @@ public class Consumer {
...
@@ -56,6 +60,7 @@ public class Consumer {
}
}
private
static
StreamingQuery
writeStreamingData
(
Dataset
<
Product
>
ds
)
throws
TimeoutException
{
private
static
StreamingQuery
writeStreamingData
(
Dataset
<
Product
>
ds
)
throws
TimeoutException
{
log
.
debug
(
"writing to CSV file"
);
return
ds
return
ds
.
writeStream
()
.
writeStream
()
.
outputMode
(
"append"
)
.
outputMode
(
"append"
)
...
@@ -66,5 +71,6 @@ public class Consumer {
...
@@ -66,5 +71,6 @@ public class Consumer {
batchDF
.
write
().
format
(
"csv"
).
mode
(
"append"
).
save
(
"src/main/resources/productDataOutPutCSV"
);
batchDF
.
write
().
format
(
"csv"
).
mode
(
"append"
).
save
(
"src/main/resources/productDataOutPutCSV"
);
})
})
.
start
();
.
start
();
}
}
}
}
src/main/java/com/nisum/spark/publisher/Publisher.java
View file @
143bc927
package
com
.
nisum
.
spark
.
publisher
;
package
com
.
nisum
.
spark
.
publisher
;
import
org.apache.spark.sql.*
;
import
org.apache.spark.sql.*
;
import
org.apache.spark.storage.StorageLevel
;
import
org.apache.spark.storage.StorageLevel
;
import
org.slf4j.Logger
;
import
org.slf4j.LoggerFactory
;
import
java.util.HashMap
;
import
java.util.HashMap
;
import
java.util.Map
;
import
java.util.Map
;
import
static
com
.
nisum
.
spark
.
util
.
Constants
.*;
import
static
com
.
nisum
.
spark
.
util
.
Constants
.*;
public
class
Publisher
{
public
class
Publisher
{
static
Logger
log
=
LoggerFactory
.
getLogger
(
Publisher
.
class
);
public
static
void
main
(
String
[]
args
)
{
public
static
void
main
(
String
[]
args
)
{
SparkSession
sparkSession
=
SparkSession
SparkSession
sparkSession
=
SparkSession
.
builder
().
master
(
"local[*]"
).
appName
(
"Spark Streaming Example"
).
getOrCreate
();
.
builder
().
master
(
"local[*]"
).
appName
(
"Spark Streaming Example"
).
getOrCreate
();
log
.
debug
(
"spark session is created"
);
Dataset
<
Row
>
dataset
=
sparkSession
.
read
().
Dataset
<
Row
>
dataset
=
sparkSession
.
read
().
format
(
"csv"
).
option
(
"header"
,
true
).
format
(
"csv"
).
option
(
"header"
,
true
).
load
(
"src/main/resources/productData.csv"
);
load
(
"src/main/resources/productData.csv"
);
...
@@ -22,12 +26,14 @@ public class Publisher {
...
@@ -22,12 +26,14 @@ public class Publisher {
}
}
private
static
void
publishCsvRecordsToKafkaTopic
(
Dataset
<
Row
>
rowDataset
)
{
private
static
void
publishCsvRecordsToKafkaTopic
(
Dataset
<
Row
>
rowDataset
)
{
if
(
null
!=
rowDataset
)
{
if
(
null
!=
rowDataset
)
{
log
.
debug
(
"reading from CSV file"
);
Dataset
<
Row
>
KafkaPublishJson
=
rowDataset
Dataset
<
Row
>
KafkaPublishJson
=
rowDataset
.
withColumn
(
"value"
,
functions
.
to_json
(
functions
.
struct
(
functions
.
col
(
"productId"
),
.
withColumn
(
"value"
,
functions
.
to_json
(
functions
.
struct
(
functions
.
col
(
"productId"
),
functions
.
col
(
"productName"
),
functions
.
col
(
"productPrice"
),
functions
.
col
(
"productName"
),
functions
.
col
(
"productPrice"
),
functions
.
col
(
"deliveryStatus"
),
functions
.
col
(
"date"
))))
functions
.
col
(
"deliveryStatus"
),
functions
.
col
(
"date"
))))
.
alias
(
"value"
).
select
(
"value"
)
.
alias
(
"value"
).
select
(
"value"
)
.
persist
(
StorageLevel
.
MEMORY_AND_DISK
());
.
persist
(
StorageLevel
.
MEMORY_AND_DISK
());
log
.
debug
(
"publishing to kafka"
);
KafkaPublishJson
KafkaPublishJson
.
write
().
format
(
"kafka"
).
options
(
KafkaCsvConfig
()).
save
();
.
write
().
format
(
"kafka"
).
options
(
KafkaCsvConfig
()).
save
();
}
}
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment