Skip to content

O
OpenAIRE-EUDAT-DMP-service-pilot
Commit 3aa7dc04 authored by Georgios Kolokythas's avatar Georgios Kolokythas
Browse files
Options

Refactors Funder, Grant and Project external fetching by adding distinct... 

Refactors Funder, Grant and Project external fetching by adding distinct values for key, indicating the source it was fetched, and it's respected display value.
parent ff9aa140
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 201 additions and 54 deletions
dmp-backend/web/src/main/java/eu/eudat/logic/builders/model/models/FunderBuilder.java
View file @ 3aa7dc04
......@@ -17,6 +17,7 @@ public class FunderBuilder extends Builder<Funder> {
private Date modified;
private Integer type;
private String source;
private String key;
public FunderBuilder id(UUID id) {
this.id = id;
......@@ -63,6 +64,11 @@ public class FunderBuilder extends Builder<Funder> {
return this;
}
public FunderBuilder key(String key) {
this.key = key;
return this;
}
@Override
public Funder build() {
Funder funder = new Funder();
......@@ -75,6 +81,7 @@ public class FunderBuilder extends Builder<Funder> {
funder.setModified(modified);
funder.setType(type);
funder.setSource(source);
funder.setKey(key);
return funder;
}
}
dmp-backend/web/src/main/java/eu/eudat/logic/builders/model/models/GrantBuilder.java
View file @ 3aa7dc04
......@@ -9,9 +9,6 @@ import java.util.Date;
import java.util.List;
import java.util.UUID;
/**
* Created by ikalyvas on 2/15/2018.
*/
public class GrantBuilder extends Builder<Grant> {
private UUID id;
......@@ -44,6 +41,8 @@ public class GrantBuilder extends Builder<Grant> {
private String source;
private String key;
public GrantBuilder id(UUID id) {
this.id = id;
return this;
......@@ -119,6 +118,11 @@ public class GrantBuilder extends Builder<Grant> {
return this;
}
public GrantBuilder key(String key) {
this.key = key;
return this;
}
@Override
public Grant build() {
Grant grant = new Grant();
......@@ -137,6 +141,7 @@ public class GrantBuilder extends Builder<Grant> {
grant.setCreationUser(creationUser);
grant.setStartDate(startDate);
grant.setSource(source);
grant.setKey(key);
return grant;
}
}
dmp-backend/web/src/main/java/eu/eudat/logic/builders/model/models/ProjectBuilder.java
View file @ 3aa7dc04
......@@ -28,6 +28,7 @@ public class ProjectBuilder extends Builder<Project> {
private Date modified;
private String description;
private String source;
private String key;
public ProjectBuilder id(UUID id) {
this.id = id;
......@@ -109,6 +110,11 @@ public class ProjectBuilder extends Builder<Project> {
return this;
}
public ProjectBuilder key(String key) {
this.key = key;
return this;
}
@Override
public Project build() {
Project project = new Project();
......@@ -127,6 +133,7 @@ public class ProjectBuilder extends Builder<Project> {
project.setCreationUser(creationUser);
project.setStartDate(startDate);
project.setSource(source);
project.setKey(key);
return project;
}
}
dmp-backend/web/src/main/java/eu/eudat/logic/managers/FunderManager.java
View file @ 3aa7dc04
......@@ -3,6 +3,7 @@ package eu.eudat.logic.managers;
import eu.eudat.data.query.items.item.funder.FunderCriteriaRequest;
import eu.eudat.logic.builders.model.models.FunderBuilder;
import eu.eudat.logic.proxy.config.ExternalUrlCriteria;
import eu.eudat.logic.proxy.config.configloaders.ConfigLoader;
import eu.eudat.logic.proxy.config.exceptions.HugeResultSet;
import eu.eudat.logic.proxy.config.exceptions.NoURLFound;
import eu.eudat.logic.proxy.fetching.RemoteFetcher;
......@@ -47,6 +48,8 @@ public class FunderManager {
eu.eudat.models.data.funder.Funder funder = apiContext.getOperationsContext().getBuilderFactory().getBuilder(FunderBuilder.class)
.reference(externalListingItem.getRemoteId()).label(externalListingItem.getName())
.status(eu.eudat.data.entities.Funder.Status.fromInteger(0))
.key(externalListingItem.getKey())
.source(externalListingItem.getTag())
.build();
if (externalListingItem.getSource() != null) {
funder.setSource(externalListingItem.getSource());
......
dmp-backend/web/src/main/java/eu/eudat/logic/managers/GrantManager.java
View file @ 3aa7dc04
......@@ -134,6 +134,7 @@ public class GrantManager {
eu.eudat.models.data.grant.Grant grant = apiContext.getOperationsContext().getBuilderFactory().getBuilder(GrantBuilder.class)
.reference(externalListingItem.getRemoteId()).label(externalListingItem.getName())
.description(externalListingItem.getDescription()).uri(externalListingItem.getUri())
.key(externalListingItem.getKey())
.abbreviation(externalListingItem.getAbbreviation()).status(eu.eudat.data.entities.Grant.Status.fromInteger(0))
.source(externalListingItem.getTag())
.build();
......
dmp-backend/web/src/main/java/eu/eudat/logic/managers/ProjectManager.java
View file @ 3aa7dc04
......@@ -48,6 +48,7 @@ public class ProjectManager {
.reference(externalListingItem.getRemoteId()).label(externalListingItem.getName())
.description(externalListingItem.getDescription()).uri(externalListingItem.getUri())
.abbreviation(externalListingItem.getAbbreviation()).status(eu.eudat.data.entities.Project.Status.fromInteger(0))
.key(externalListingItem.getKey())
.source(externalListingItem.getTag())
.build();
......
dmp-backend/web/src/main/java/eu/eudat/logic/proxy/fetching/RemoteFetcher.java
View file @ 3aa7dc04
......@@ -129,7 +129,7 @@ public class RemoteFetcher {
for (UrlConfiguration urlConfig : urlConfigs) {
ifFunderQueryExist(urlConfig, externalUrlCriteria);
if (urlConfig.getType() == null || urlConfig.getType().equals("External")) {
results.addAll(getAllResultsFromUrl(urlConfig.getUrl(), fetchStrategy, urlConfig.getData(), urlConfig.getPaginationPath(), externalUrlCriteria, urlConfig.getLabel(), urlConfig.getContentType(), urlConfig.getFirstpage()));
results.addAll(getAllResultsFromUrl(urlConfig.getUrl(), fetchStrategy, urlConfig.getData(), urlConfig.getPaginationPath(), externalUrlCriteria, urlConfig.getLabel(), urlConfig.getKey(), urlConfig.getContentType(), urlConfig.getFirstpage()));
} else if (urlConfig.getType() != null && urlConfig.getType().equals("Internal")) {
results.addAll(getAllResultsFromMockUpJson(urlConfig.getUrl(), externalUrlCriteria.getLike()));
}
......@@ -184,14 +184,14 @@ public class RemoteFetcher {
return completedPath;
}
private List<Map<String, String>> getAllResultsFromUrl(String path, FetchStrategy fetchStrategy, final DataUrlConfiguration jsonDataPath, final String jsonPaginationPath, ExternalUrlCriteria externalUrlCriteria, String key, String contentType, String firstPage) throws HugeResultSet {
private List<Map<String, String>> getAllResultsFromUrl(String path, FetchStrategy fetchStrategy, final DataUrlConfiguration jsonDataPath, final String jsonPaginationPath, ExternalUrlCriteria externalUrlCriteria, String tag, String key, String contentType, String firstPage) throws HugeResultSet {
Set<Integer> pages = new HashSet<>();
String replacedPath = replaceCriteriaOnUrl(path, externalUrlCriteria, firstPage);
Results results = getResultsFromUrl(replacedPath, jsonDataPath, jsonPaginationPath, contentType);
if (fetchStrategy == FetchStrategy.FIRST)
return results == null ? new LinkedList<>() : results.getResults().stream().peek(x -> x.put("tag", key)).collect(Collectors.toList());
return results == null ? new LinkedList<>() : results.getResults().stream().peek(x -> x.put("tag", tag)).peek(x -> x.put("key", key)).collect(Collectors.toList());
if (results != null && results.getPagination() != null && results.getPagination().get("pages") != null) //if has more pages, add them to the pages set
for (int i = 2; i <= results.getPagination().get("pages"); i++)
......@@ -210,7 +210,7 @@ public class RemoteFetcher {
Results remainingResults = optionalResults.orElseGet(Results::new);
remainingResults.getResults().addAll(results.getResults());
return remainingResults.getResults().stream().peek(x -> x.put("tag", key)).collect(Collectors.toList());
return remainingResults.getResults().stream().peek(x -> x.put("tag", tag)).collect(Collectors.toList());
}
......@@ -271,8 +271,7 @@ public class RemoteFetcher {
String filePath = Paths.get(path).toUri().toURL().toString();
ObjectMapper mapper = new ObjectMapper();
internalResults = mapper.readValue(new File(filePath), new TypeReference<List<Map<String, Object>>>(){});
searchListMap(internalResults, query);
return internalResults;
return searchListMap(internalResults, query);
} catch (Exception e) {
e.printStackTrace();
return new LinkedList<>();
......
dmp-backend/web/src/main/java/eu/eudat/models/data/external/FundersExternalSourcesModel.java
View file @ 3aa7dc04
......@@ -15,6 +15,7 @@ public class FundersExternalSourcesModel extends ExternalListingItem<FundersExte
model.setDescription(item.get("description"));
model.setSource(item.get("source"));
model.setTag(item.get("tag"));
model.setKey(item.get("key"));
this.add(model);
}
return this;
......
dmp-backend/web/src/main/java/eu/eudat/models/data/external/GrantsExternalSourcesModel.java
View file @ 3aa7dc04
......@@ -14,6 +14,7 @@ public class GrantsExternalSourcesModel extends ExternalListingItem<GrantsExtern
model.setName(item.get("name"));
model.setDescription(item.get("description"));
model.setTag(item.get("tag"));
model.setKey(item.get("key"));
this.add(model);
}
return this;
......
dmp-backend/web/src/main/java/eu/eudat/models/data/external/ProjectsExternalSourcesModel.java
View file @ 3aa7dc04
......@@ -51,6 +51,13 @@ public class ProjectsExternalSourcesModel extends ExternalListingItem<ProjectsEx
model.setTag(item.get("tag"));
}
JsonNode key = node.get("key");
if (key != null && !key.isNull() && key.isObject()) {
model.setKey(node.get("key").get("$").asText());
} else {
model.setKey(item.get("key"));
}
this.add(model);
} catch (IOException e) {
......
dmp-backend/web/src/main/java/eu/eudat/models/data/funder/Funder.java
View file @ 3aa7dc04
......@@ -16,6 +16,7 @@ public class Funder implements DataModel<eu.eudat.data.entities.Funder, Funder>
private Date modified;
private Integer type;
private String source;
private String key;
public UUID getId() {
return id;
......@@ -80,6 +81,13 @@ public class Funder implements DataModel<eu.eudat.data.entities.Funder, Funder>
this.source = source;
}
public String getKey() {
return key;
}
public void setKey(String key) {
this.key = key;
}
@Override
public Funder fromDataModel(eu.eudat.data.entities.Funder entity) {
this.id = entity.getId();
......@@ -93,9 +101,9 @@ public class Funder implements DataModel<eu.eudat.data.entities.Funder, Funder>
if (entity.getReference() != null) {
String source = entity.getReference().substring(0, entity.getReference().indexOf(":"));
if (source.equals("dmp")) {
this.source = "Internal";
this.key = "Internal";
} else {
this.source = source;
this.key = source;
}
}
......@@ -105,19 +113,26 @@ public class Funder implements DataModel<eu.eudat.data.entities.Funder, Funder>
@Override
public eu.eudat.data.entities.Funder toDataModel() {
eu.eudat.data.entities.Funder entity = new eu.eudat.data.entities.Funder();
entity.setId(UUID.randomUUID());
if (this.getId() != null) {
entity.setId(this.getId());
} else {
entity.setId(UUID.randomUUID());
}
entity.setLabel(this.label);
entity.setType(this.type);
if (this.source != null && this.source.equals("Internal")) this.source = "dmp";
if (this.source == null && this.reference != null && this.reference.startsWith("dmp:")) {
entity.setReference(this.reference);
// If internal, key has no value, fill it.
if ((this.source != null && this.source.equals("Internal")) || (this.key != null && this.key.equals("Internal"))) this.key = "dmp";
// Logic for the different "key" cases.
if ((this.key == null || this.key.trim().isEmpty()) && (this.source != null && !this.source.trim().isEmpty())) {
this.key = this.source;
}
if (this.reference != null && !this.reference.trim().isEmpty()
&& this.source != null && !this.source.trim().isEmpty()) {
if (this.source.equals(this.reference.substring(0, this.source.length()))) {
if (this.reference != null && this.reference.startsWith("dmp:")) {
entity.setReference(this.reference);
} else if (this.reference != null && !this.reference.trim().isEmpty() && this.key != null && !this.key.trim().isEmpty()) {
if (this.key.equals(this.reference.substring(0, this.key.length()))) {
entity.setReference(this.reference);
} else {
entity.setReference(this.source.toLowerCase() + ":" + this.reference);
entity.setReference(this.key.toLowerCase() + ":" + this.reference);
}
}
entity.setDefinition(this.definition);
......
dmp-backend/web/src/main/java/eu/eudat/models/data/grant/Grant.java
View file @ 3aa7dc04
......@@ -32,6 +32,7 @@ public class Grant implements DataModel<eu.eudat.data.entities.Grant, Grant> {
private List<ContentFile> files;
private UUID funderId;
private String source;
private String key;
public UUID getId() {
return id;
......@@ -159,6 +160,13 @@ public class Grant implements DataModel<eu.eudat.data.entities.Grant, Grant> {
this.source = source;
}
public String getKey() {
return key;
}
public void setKey(String key) {
this.key = key;
}
@Override
public Grant fromDataModel(eu.eudat.data.entities.Grant entity) {
this.id = entity.getId();
......@@ -180,9 +188,9 @@ public class Grant implements DataModel<eu.eudat.data.entities.Grant, Grant> {
if (entity.getReference() != null) {
String source = entity.getReference().substring(0, entity.getReference().indexOf(":"));
if (source.equals("dmp")) {
this.source = "Internal";
this.key = "Internal";
} else {
this.source = source;
this.key = source;
}
}
......@@ -192,20 +200,22 @@ public class Grant implements DataModel<eu.eudat.data.entities.Grant, Grant> {
@Override
public eu.eudat.data.entities.Grant toDataModel() {
eu.eudat.data.entities.Grant entity = new eu.eudat.data.entities.Grant();
entity.setId(UUID.randomUUID());
if (this.getId() != null) {
entity.setId(this.getId());
} else {
entity.setId(UUID.randomUUID());
}
entity.setAbbreviation(this.abbreviation);
entity.setLabel(this.label);
entity.setType(this.type);
if (this.source != null && this.source.equals("Internal")) this.source = "dmp";
if (this.source == null && this.reference != null && this.reference.startsWith("dmp:")) {
if ((this.source != null && this.source.equals("Internal")) || (this.key != null && this.key.equals("Internal"))) this.key = "dmp";
if (this.reference != null && this.reference.startsWith("dmp:")) {
entity.setReference(this.reference);
}
if (this.reference != null && !this.reference.trim().isEmpty()
&& this.source != null && !this.source.trim().isEmpty()) {
if (this.source.equals(this.reference.substring(0, this.source.length()))) {
} else if (this.reference != null && !this.reference.trim().isEmpty() && this.key != null && !this.key.trim().isEmpty()) {
if (this.key.equals(this.reference.substring(0, this.key.length()))) {
entity.setReference(this.reference);
} else {
entity.setReference(this.source.toLowerCase() + ":" + this.reference);
entity.setReference(this.key.toLowerCase() + ":" + this.reference);
}
}
entity.setUri(this.uri);
......
dmp-backend/web/src/main/java/eu/eudat/models/data/project/Project.java
View file @ 3aa7dc04
......@@ -29,6 +29,7 @@ public class Project implements DataModel<eu.eudat.data.entities.Project, Projec
private String description;
private List<ContentFile> files;
private String source;
private String key;
public UUID getId() {
return id;
......@@ -149,6 +150,13 @@ public class Project implements DataModel<eu.eudat.data.entities.Project, Projec
this.source = source;
}
public String getKey() {
return key;
}
public void setKey(String key) {
this.key = key;
}
@Override
public Project fromDataModel(eu.eudat.data.entities.Project entity) {
this.id = entity.getId();
......@@ -168,9 +176,9 @@ public class Project implements DataModel<eu.eudat.data.entities.Project, Projec
if (entity.getReference() != null) {
String source = entity.getReference().substring(0, entity.getReference().indexOf(":"));
if (source.equals("dmp")) {
this.source = "Internal";
this.key = "Internal";
} else {
this.source = source;
this.key = source;
}
}
......@@ -184,16 +192,14 @@ public class Project implements DataModel<eu.eudat.data.entities.Project, Projec
entity.setAbbreviation(this.abbreviation);
entity.setLabel(this.label);
entity.setType(this.type);
if (this.source != null && this.source.equals("Internal")) this.source = "dmp";
if (this.source == null && this.reference != null && this.reference.startsWith("dmp:")) {
if ((this.source != null && this.source.equals("Internal")) || (this.key != null && this.key.equals("Internal"))) this.key = "dmp";
if (this.reference != null && this.reference.startsWith("dmp:")) {
entity.setReference(this.reference);
}
if (this.reference != null && !this.reference.trim().isEmpty()
&& this.source != null && !this.source.trim().isEmpty()) {
if (this.source.equals(this.reference.substring(0, this.source.length()))) {
} else if (this.reference != null && !this.reference.trim().isEmpty() && this.key != null && !this.key.trim().isEmpty()) {
if (this.key.equals(this.reference.substring(0, this.key.length()))) {
entity.setReference(this.reference);
} else {
entity.setReference(this.source.toLowerCase() + ":" + this.reference);
entity.setReference(this.key.toLowerCase() + ":" + this.reference);
}
}
entity.setUri(this.uri);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment