Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
What's new
10
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Open sidebar
massive-geo-data
storage-system
Commits
e6925a3a
Commit
e6925a3a
authored
Jun 04, 2018
by
Álex Cortiñas
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Some refinements
parent
fd75354c
Changes
5
Hide whitespace changes
Inline
Side-by-side
Showing
5 changed files
with
23 additions
and
14 deletions
+23
-14
src/main/java/es/udc/lbd/gis/storagesystem/StorageSystem.java
...main/java/es/udc/lbd/gis/storagesystem/StorageSystem.java
+11
-11
src/main/java/es/udc/lbd/gis/storagesystem/conectors/KafkaReader.java
...a/es/udc/lbd/gis/storagesystem/conectors/KafkaReader.java
+1
-1
src/main/java/es/udc/lbd/gis/storagesystem/model/Event.java
src/main/java/es/udc/lbd/gis/storagesystem/model/Event.java
+1
-0
src/main/resources/config.properties
src/main/resources/config.properties
+5
-1
src/main/resources/datasource.json
src/main/resources/datasource.json
+5
-1
No files found.
src/main/java/es/udc/lbd/gis/storagesystem/StorageSystem.java
View file @
e6925a3a
...
...
@@ -22,12 +22,12 @@ public class StorageSystem {
MyProperties
prop
=
new
MyProperties
(
"/config.properties"
);
PostgresWriter
pg
=
new
PostgresWriter
(
prop
.
get
(
"datasource"
),
prop
.
get
(
"pg-ip-port"
),
prop
.
get
(
"pg-database"
),
prop
.
get
(
"pg-user"
),
prop
.
get
(
"pg-pass"
));
//
PostgresWriter pg = new PostgresWriter(
//
prop.get("datasource"),
//
prop.get("pg-ip-port"),
//
prop.get("pg-database"),
//
prop.get("pg-user"),
//
prop.get("pg-pass"));
DruidWriter
druidWriter
=
new
DruidWriter
(
prop
.
get
(
"druidSpec"
),
...
...
@@ -39,26 +39,26 @@ public class StorageSystem {
Boolean
.
parseBoolean
(
prop
.
get
(
"commit"
)),
prop
.
get
(
"topic"
));
Function
<
Long
,
Boolean
>
notOld
=
(
time
)
->
time
>
System
.
currentTimeMillis
()
-
Long
.
parseLong
(
prop
.
get
(
"kafka-period"
));
//
Function<Long, Boolean> notOld = (time) ->
//
time > System.currentTimeMillis() - Long.parseLong(prop.get("kafka-period"));
try
{
kafka
.
read
((
List
<
ConsumerRecord
<
String
,
String
>>
buffer
)
->
{
List
<
Event
>
events
=
buffer
.
stream
()
.
map
(
record
->
ModelMapper
.
toEvent
(
record
.
value
()).
complete
())
.
filter
(
m
->
notOld
.
apply
(
m
.
getTime
()))
//
.filter(m -> notOld.apply(m.getTime()))
.
collect
(
Collectors
.
toList
());
if
(
events
.
size
()
>
0
)
{
pg
.
insert
(
events
);
//
pg.insert(events);
druidWriter
.
insert
(
events
);
}
return
null
;
});
}
finally
{
pg
.
disconnect
();
//
pg.disconnect();
druidWriter
.
disconnect
();
kafka
.
disconnect
();
}
...
...
src/main/java/es/udc/lbd/gis/storagesystem/conectors/KafkaReader.java
View file @
e6925a3a
...
...
@@ -15,7 +15,7 @@ import org.slf4j.LoggerFactory;
public
class
KafkaReader
{
private
static
final
Logger
logger
=
LoggerFactory
.
getLogger
(
KafkaReader
.
class
);
private
static
final
int
maxPoll
=
5
000
;
private
static
final
int
maxPoll
=
1
000
;
private
static
final
int
intervalCommit
=
60000
;
private
static
Properties
props
=
null
;
...
...
src/main/java/es/udc/lbd/gis/storagesystem/model/Event.java
View file @
e6925a3a
...
...
@@ -61,6 +61,7 @@ public class Event {
this
.
y_7
=
df7
.
format
(
this
.
y
);
this
.
x_8
=
df8
.
format
(
this
.
x
);
this
.
y_8
=
df8
.
format
(
this
.
y
);
this
.
time
=
System
.
currentTimeMillis
();
return
this
;
}
...
...
src/main/resources/config.properties
View file @
e6925a3a
# datasource (druid
and
postgres table)
# datasource (druid
,
postgres table
and mongo collection
)
datasource
=
theTopic
# kafka topic
...
...
@@ -18,3 +18,7 @@ commit=true
# time the data is valid, usually depending on datasource.json
kafka-period
=
540000
mongo-host
=
127.0.0.1
mongo-port
=
27017
mongo-database
=
database
src/main/resources/datasource.json
View file @
e6925a3a
...
...
@@ -6,7 +6,11 @@
},
"dataSources"
:
[
{
"spec"
:
{
"properties"
:
{
"task.partitions"
:
"1"
,
"task.replicants"
:
"1"
},
"spec"
:
{
"dataSchema"
:
{
"dataSource"
:
"theTopic"
,
"granularitySpec"
:
{
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment