Skip to content
Commits on Source (69)
......@@ -12,7 +12,7 @@ output {
password =>
index =>"opendmp.logs"
#manage_template => true
#template => "/usr/share/logstash/templates/audit/cite_elas_openDMP.json"
#template => "/usr/share/logstash/templates/audit/openDMP.json"
#template_name => "cite.elas.openDMP-audit*"
#template_overwrite => true
}
......
PROFILE=staging
\ No newline at end of file
FROM openjdk:8-jdk-alpine
RUN apk add --update \
curl \
&& rm -rf /var/cache/apk/*
VOLUME /tmp
ARG PROFILE=production
ENV PROF $PROFILE
ADD web/src/main/resources/ProjectConfiguration.xml /tmp/ProjectConfiguration.xml
ADD web/src/main/resources/ExternalUrls.xml /tmp/ExternalUrls.xml
ADD web/target/web-1.0-SNAPSHOT.jar app.jar
ENTRYPOINT ["java","-Djava.security.egd=file:/dev/./urandom" ,"-Dspring.profiles.active=${PROF}","-jar","/app.jar"]
\ No newline at end of file
FROM maven:3-jdk-8-alpine AS MAVEN_BUILD
COPY pom.xml /build/
COPY data /build/data/
COPY elastic /build/elastic/
COPY logging /build/logging/
COPY queryable /build/queryable/
COPY web /build/web/
WORKDIR /build/
RUN mvn package
FROM openjdk:8-jre-alpine
WORKDIR /app
COPY --from=MAVEN_BUILD /build/web/target/web-1.0-SNAPSHOT.jar /app.jar
ENTRYPOINT ["java","-Djava.security.egd=file:/dev/./urandom" ,"-Dspring.profiles.active=${PROFILE}","-jar","/app.jar"]
\ No newline at end of file
package eu.eudat.data.dao.criteria;
import eu.eudat.data.entities.DMP;
import eu.eudat.data.entities.Grant;
import java.util.Date;
......@@ -20,6 +19,8 @@ public class DataManagementPlanCriteria extends Criteria<DMP> {
private List<UUID> collaborators;
private List<UUID> datasetTemplates;
private boolean isPublic;
private boolean onlyPublic;
private Short grantStatus;
public Date getPeriodStart() {
return periodStart;
......@@ -97,4 +98,20 @@ public class DataManagementPlanCriteria extends Criteria<DMP> {
public void setIsPublic(boolean isPublic) {
this.isPublic = isPublic;
}
public boolean isOnlyPublic() {
return onlyPublic;
}
public void setOnlyPublic(boolean onlyPublic) {
this.onlyPublic = onlyPublic;
}
public Short getGrantStatus() {
return grantStatus;
}
public void setGrantStatus(Short grantStatus) {
this.grantStatus = grantStatus;
}
}
......@@ -33,6 +33,7 @@ public class DatasetProfileCriteria extends Criteria<DatasetProfile> {
private Short filter;
private UUID userId;
private boolean finalized;
private Integer status;
public boolean getAllVersions() { return allVersions; }
public void setAllVersions(boolean allVersions) { this.allVersions = allVersions; }
......@@ -60,4 +61,12 @@ public class DatasetProfileCriteria extends Criteria<DatasetProfile> {
public void setFinalized(boolean finalized) {
this.finalized = finalized;
}
public Integer getStatus() {
return status;
}
public void setStatus(Integer status) {
this.status = status;
}
}
......@@ -9,6 +9,7 @@ import eu.eudat.data.entities.UserInfo;
import eu.eudat.queryable.QueryableList;
import eu.eudat.queryable.types.FieldSelectionType;
import eu.eudat.queryable.types.SelectionField;
import eu.eudat.types.grant.GrantStateType;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Async;
import org.springframework.stereotype.Component;
......@@ -16,6 +17,7 @@ import org.springframework.stereotype.Component;
import javax.persistence.criteria.Join;
import javax.persistence.criteria.JoinType;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
......@@ -73,6 +75,14 @@ public class DMPDaoImpl extends DatabaseAccess<DMP> implements DMPDao {
if (criteria.getDatasetTemplates() != null && !criteria.getDatasetTemplates().isEmpty()) {
query.where((builder, root) -> root.join("associatedDmps", JoinType.LEFT).get("id").in(criteria.getDatasetTemplates()));
}
if (criteria.getGrantStatus() != null) {
if (criteria.getGrantStatus().equals(GrantStateType.FINISHED.getValue().shortValue()))
query.where((builder, root) -> builder.lessThan(root.get("grant").get("enddate"), new Date()));
if (criteria.getGrantStatus().equals(GrantStateType.ONGOING.getValue().shortValue()))
query.where((builder, root) ->
builder.or(builder.greaterThan(root.get("grant").get("enddate"), new Date())
, builder.isNull(root.get("grant").get("enddate"))));
}
query.where((builder, root) -> builder.notEqual(root.get("status"), DMP.DMPStatus.DELETED.getValue()));
return query;
}
......
......@@ -52,6 +52,9 @@ public class DatasetProfileDaoImpl extends DatabaseAccess<DatasetProfile> implem
builder.notEqual(root.get("id"), criteria.getUserId())));
}
}
if (criteria.getStatus() != null) {
query.where(((builder, root) -> builder.equal(root.get("status"), criteria.getStatus())));
}
if (criteria.getFinalized()) {
query.where(((builder, root) -> builder.equal(root.get("status"), DatasetProfile.Status.FINALIZED.getValue())));
} else {
......
package eu.eudat.data.dao.entities;
import eu.eudat.data.dao.DatabaseAccessLayer;
import eu.eudat.data.entities.DoiFunder;
import java.util.UUID;
public interface DoiFunderDao extends DatabaseAccessLayer<DoiFunder, UUID> {
DoiFunder findFunderByName(String name);
}
package eu.eudat.data.dao.entities;
import eu.eudat.data.dao.DatabaseAccess;
import eu.eudat.data.dao.databaselayer.service.DatabaseService;
import eu.eudat.data.entities.DoiFunder;
import eu.eudat.queryable.QueryableList;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
@Component("DoiFunderDao")
public class DoiFunderDaoImpl extends DatabaseAccess<DoiFunder> implements DoiFunderDao {
@Autowired
public DoiFunderDaoImpl(DatabaseService<DoiFunder> databaseService) {
super(databaseService);
}
@Override
public DoiFunder findFunderByName(String name) {
return this.asQueryable().toList().stream().filter(doiFunder -> name.contains(doiFunder.getName()) || doiFunder.getName().contains(name)).findFirst().orElse(null);
}
@Override
public DoiFunder createOrUpdate(DoiFunder item) {
return this.getDatabaseService().createOrUpdate(item, DoiFunder.class);
}
@Override
public CompletableFuture<DoiFunder> createOrUpdateAsync(DoiFunder item) {
return CompletableFuture.supplyAsync(() -> this.createOrUpdate(item));
}
@Override
public DoiFunder find(UUID id) {
return this.getDatabaseService().getQueryable(DoiFunder.class).where(((builder, root) -> builder.equal(root.get("id"), id))).getSingle();
}
@Override
public DoiFunder find(UUID id, String hint) {
throw new UnsupportedOperationException();
}
@Override
public void delete(DoiFunder item) {
this.getDatabaseService().delete(item);
}
@Override
public QueryableList<DoiFunder> asQueryable() {
return this.getDatabaseService().getQueryable(DoiFunder.class);
}
}
package eu.eudat.data.dao.entities;
import eu.eudat.data.dao.DatabaseAccessLayer;
import eu.eudat.data.entities.UserAssociation;
import eu.eudat.data.entities.UserInfo;
import java.util.List;
import java.util.UUID;
public interface UserAssociationDao extends DatabaseAccessLayer<UserAssociation, UUID> {
public List<UserAssociation> getAssociated(UserInfo userId);
public Boolean areAssociated(UserInfo firstUser, UserInfo secondUser);
}
package eu.eudat.data.dao.entities;
import eu.eudat.data.dao.DatabaseAccess;
import eu.eudat.data.dao.databaselayer.service.DatabaseService;
import eu.eudat.data.entities.UserAssociation;
import eu.eudat.data.entities.UserInfo;
import eu.eudat.queryable.QueryableList;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.CompletableFuture;
@Component("UserAssociationDao")
public class UserAssociationDaoImpl extends DatabaseAccess<UserAssociation> implements UserAssociationDao {
@Autowired
public UserAssociationDaoImpl(DatabaseService<UserAssociation> databaseService) {
super(databaseService);
}
@Override
public UserAssociation createOrUpdate(UserAssociation item) {
return this.getDatabaseService().createOrUpdate(item, UserAssociation.class);
}
@Override
public CompletableFuture<UserAssociation> createOrUpdateAsync(UserAssociation item) {
return CompletableFuture.supplyAsync(() -> this.createOrUpdate(item));
}
@Override
public UserAssociation find(UUID id) {
return this.getDatabaseService().getQueryable(UserAssociation.class).where(((builder, root) -> builder.equal(root.get("id"), id))).getSingle();
}
@Override
public UserAssociation find(UUID id, String hint) {
throw new UnsupportedOperationException();
}
@Override
public void delete(UserAssociation item) {
this.getDatabaseService().delete(item);
}
@Override
public QueryableList<UserAssociation> asQueryable() {
return this.getDatabaseService().getQueryable(UserAssociation.class);
}
@Override
public List<UserAssociation> getAssociated(UserInfo userId) {
return this.getDatabaseService().getQueryable(UserAssociation.class).where(((builder, root) ->
builder.or(builder.equal(root.get("firstUser"), userId), builder.equal(root.get("secondUser"), userId)))).toList();
}
@Override
public Boolean areAssociated(UserInfo firstUser, UserInfo secondUser) {
return this.getDatabaseService().getQueryable(UserAssociation.class).where(((builder, root) ->
builder.or(
builder.and(
builder.equal(root.get("firstUser"), firstUser),
builder.equal(root.get("secondUser"), secondUser)
),
builder.and(
builder.equal(root.get("secondUser"), firstUser),
builder.equal(root.get("firstUser"), secondUser)
)
))).count() > 0;
}
}
......@@ -132,6 +132,9 @@ public class DataRepository implements Serializable, DataEntity<DataRepository,
@Override
public void update(DataRepository entity) {
this.label = entity.getLabel();
this.abbreviation = entity.getAbbreviation();
this.uri = entity.getUri();
}
......
package eu.eudat.data.entities;
import eu.eudat.queryable.queryableentity.DataEntity;
import javax.persistence.*;
import java.util.List;
import java.util.UUID;
@Entity
@Table(name = "\"DoiFunder\"")
public class DoiFunder implements DataEntity<DoiFunder, UUID> {
@Id
private UUID id;
@Column(name = "name")
private String name;
@Column(name = "doi")
private String doi;
public UUID getId() {
return id;
}
public void setId(UUID id) {
this.id = id;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getDoi() {
return doi;
}
public void setDoi(String doi) {
this.doi = doi;
}
@Override
public void update(DoiFunder entity) {
this.name = entity.name;
this.doi = entity.doi;
}
@Override
public UUID getKeys() {
return id;
}
@Override
public DoiFunder buildFromTuple(List<Tuple> tuple, List<String> fields, String base) {
return null;
}
}
......@@ -139,7 +139,9 @@ public class Registry implements DataEntity<Registry, UUID> {
@Override
public void update(Registry entity) {
this.label = entity.getLabel();
this.abbreviation = entity.getAbbreviation();
this.uri = entity.getUri();
}
@Override
......
......@@ -134,7 +134,9 @@ public class Service implements DataEntity<Service, UUID> {
@Override
public void update(Service entity) {
this.label = entity.getLabel();
this.abbreviation = entity.getAbbreviation();
this.uri = entity.getUri();
}
@Override
......
package eu.eudat.data.entities;
import eu.eudat.queryable.queryableentity.DataEntity;
import org.hibernate.annotations.GenericGenerator;
import javax.persistence.*;
import java.util.List;
import java.util.UUID;
@Entity
@Table(name = "\"UserAssociation\"")
public class UserAssociation implements DataEntity<UserAssociation, UUID> {
@Id
@GeneratedValue
@GenericGenerator(name = "uuid2", strategy = "uuid2")
@Column(name = "id", updatable = false, nullable = false, columnDefinition = "BINARY(16)")
private UUID id;
@OneToOne(fetch = FetchType.LAZY)
@JoinColumn(name = "\"firstUser\"")
private UserInfo firstUser;
@OneToOne(fetch = FetchType.LAZY)
@JoinColumn(name = "\"secondUser\"")
private UserInfo secondUser;
public UUID getId() {
return id;
}
public void setId(UUID id) {
this.id = id;
}
public UserInfo getFirstUser() {
return firstUser;
}
public void setFirstUser(UserInfo firstUser) {
this.firstUser = firstUser;
}
public UserInfo getSecondUser() {
return secondUser;
}
public void setSecondUser(UserInfo secondUser) {
this.secondUser = secondUser;
}
@Override
public void update(UserAssociation entity) {
}
@Override
public UUID getKeys() {
return null;
}
@Override
public UserAssociation buildFromTuple(List<Tuple> tuple, List<String> fields, String base) {
return null;
}
}
package eu.eudat.elastic.criteria;
import java.util.List;
import java.util.UUID;
public class DmpCriteria extends Criteria {
private String like;
private Short status;
private List<UUID> templates;
private List<UUID> grants;
private List<UUID> collaborators;
private List<UUID> organizations;
private boolean isPublic;
private List<UUID> groupIds;
private boolean allowAllVersions;
private Short grantStatus;
public String getLike() {
return like;
}
public void setLike(String like) {
this.like = like;
}
public Short getStatus() {
return status;
}
public void setStatus(Short status) {
this.status = status;
}
public List<UUID> getTemplates() {
return templates;
}
public void setTemplates(List<UUID> templates) {
this.templates = templates;
}
public List<UUID> getGrants() {
return grants;
}
public void setGrants(List<UUID> grants) {
this.grants = grants;
}
public List<UUID> getCollaborators() {
return collaborators;
}
public void setCollaborators(List<UUID> collaborators) {
this.collaborators = collaborators;
}
public List<UUID> getOrganizations() {
return organizations;
}
public void setOrganizations(List<UUID> organizations) {
this.organizations = organizations;
}
public boolean isPublic() {
return isPublic;
}
public void setPublic(boolean aPublic) {
isPublic = aPublic;
}
public List<UUID> getGroupIds() {
return groupIds;
}
public void setGroupIds(List<UUID> groupIds) {
this.groupIds = groupIds;
}
public boolean isAllowAllVersions() {
return allowAllVersions;
}
public void setAllowAllVersions(boolean allowAllVersions) {
this.allowAllVersions = allowAllVersions;
}
public Short getGrantStatus() {
return grantStatus;
}
public void setGrantStatus(Short grantStatus) {
this.grantStatus = grantStatus;
}
}
package eu.eudat.elastic.entities;
import org.elasticsearch.common.xcontent.XContentBuilder;
import java.io.IOException;
import java.util.Map;
import java.util.UUID;
public class DatasetTempalate implements ElasticEntity<DatasetTempalate> {
private UUID id;
private String name;
public UUID getId() {
return id;
}
public void setId(UUID id) {
this.id = id;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
@Override
public XContentBuilder toElasticEntity(XContentBuilder builder) throws IOException {
builder.startObject();
builder.field("id", this.id.toString());
builder.field("name", this.name);
builder.endObject();
return builder;
}
@Override
public DatasetTempalate fromElasticEntity(Map<String, Object> fields) {
this.id = UUID.fromString((String) fields.get("id"));
this.name = (String) fields.get("name");
return this;
}
}
package eu.eudat.elastic.entities;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.stream.Collectors;
public class Dmp implements ElasticEntity<Dmp> {
private static final Logger logger = LoggerFactory.getLogger(Dmp.class);
public enum DMPStatus {
ACTIVE((short) 0), FINALISED((short) 1),DELETED((short) 99);
private short value;
private DMPStatus(short value) {
this.value = value;
}
public short getValue() {
return value;
}
public static DMPStatus fromInteger(short value) {
switch (value) {
case 0:
return ACTIVE;
case 1:
return FINALISED;
case 99:
return DELETED;
default:
throw new RuntimeException("Unsupported DMP Status");
}
}
}
private UUID id;
private String label;
private String description;
private UUID groupId;
private Short status;
private List<DatasetTempalate> templates;
private List<Collaborator> collaborators;
private List<Organization> organizations;
private Boolean lastVersion;
private Boolean lastPublicVersion;
private Boolean isPublic;
private List<Dataset> datasets;
private UUID grant;
private Short grantStatus;
public UUID getId() {
return id;
}
public void setId(UUID id) {
this.id = id;
}
public String getLabel() {
return label;
}
public void setLabel(String label) {
this.label = label;
}
public String getDescription() {
return description;
}
public void setDescription(String description) {
this.description = description;
}
public UUID getGroupId() {
return groupId;
}
public void setGroupId(UUID groupId) {
this.groupId = groupId;
}
public Short getStatus() {
return status;
}
public void setStatus(Short status) {
this.status = status;
}
public List<DatasetTempalate> getTemplates() {
return templates;
}
public void setTemplates(List<DatasetTempalate> templates) {
this.templates = templates;
}
public List<Collaborator> getCollaborators() {
return collaborators;
}
public void setCollaborators(List<Collaborator> collaborators) {
this.collaborators = collaborators;
}
public List<Organization> getOrganizations() {
return organizations;
}
public void setOrganizations(List<Organization> organizations) {
this.organizations = organizations;
}
public Boolean getLastVersion() {
return lastVersion;
}
public void setLastVersion(Boolean lastVersion) {
this.lastVersion = lastVersion;
}
public Boolean getLastPublicVersion() {
return lastPublicVersion;
}
public void setLastPublicVersion(Boolean lastPublicVersion) {
this.lastPublicVersion = lastPublicVersion;
}
public Boolean getPublic() {
return isPublic;
}
public void setPublic(Boolean aPublic) {
isPublic = aPublic;
}
public List<Dataset> getDatasets() {
return datasets;
}
public void setDatasets(List<Dataset> datasets) {
this.datasets = datasets;
}
public UUID getGrant() {
return grant;
}
public void setGrant(UUID grant) {
this.grant = grant;
}
public Short getGrantStatus() {
return grantStatus;
}
public void setGrantStatus(Short grantStatus) {
this.grantStatus = grantStatus;
}
@Override
public XContentBuilder toElasticEntity(XContentBuilder builder) throws IOException {
builder.startObject();
if (this.id != null) {
builder.field(MapKey.ID.getName(), this.id.toString());
}
builder.field(MapKey.LABEL.getName(), this.label);
builder.field(MapKey.DESCRIPTION.getName(), this.description);
if (this.groupId != null) {
builder.field(MapKey.GROUPID.getName(), this.groupId.toString());
}
builder.field(MapKey.STATUS.getName(), this.status);
if (this.templates != null && !this.templates.isEmpty()) {
builder.startArray(MapKey.TEMPLATES.getName());
this.templates.forEach(template -> {
try {
template.toElasticEntity(builder);
} catch (IOException e) {
logger.error(e.getMessage(), e);
}
});
builder.endArray();
}
if (this.collaborators != null && !this.collaborators.isEmpty()) {
builder.startArray(MapKey.COLLABORATORS.getName());
this.collaborators.forEach(collaborator -> {
try {
collaborator.toElasticEntity(builder);
} catch (IOException e) {
logger.error(e.getMessage(), e);
}
});
builder.endArray();
}
if (this.organizations != null && !this.organizations.isEmpty()) {
builder.startArray(MapKey.ORGANIZATIONS.getName());
this.organizations.forEach(organization -> {
try {
organization.toElasticEntity(builder);
} catch (IOException e) {
logger.error(e.getMessage(), e);
}
});
builder.endArray();
}
builder.field(MapKey.LASTVERSION.getName(), this.lastVersion);
builder.field(MapKey.LASTPUBLICVERSION.getName(), this.lastPublicVersion);
builder.field(MapKey.ISPUBLIC.getName(), this.isPublic);
if (datasets != null && !this.datasets.isEmpty()) {
builder.startArray(MapKey.DATASETS.getName());
this.datasets.forEach(dataset -> {
try {
if (dataset != null) {
dataset.toElasticEntity(builder);
}
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
});
builder.endArray();
}
if (this.grant != null) {
builder.field(MapKey.GRANT.getName(), this.grant.toString());
}
builder.field(MapKey.GRANTSTATUS.getName(), this.grantStatus);
builder.endObject();
return builder;
}
@Override
public Dmp fromElasticEntity(Map<String, Object> fields) {
this.id = UUID.fromString((String) fields.get(MapKey.ID.getName()));
this.label = (String) fields.get(MapKey.LABEL.getName());
this.description = (String) fields.get(MapKey.DESCRIPTION.getName());
if (fields.get(MapKey.GROUPID.getName()) != null) {
this.groupId = UUID.fromString((String) fields.get(MapKey.GROUPID.getName()));
}
this.status = Short.valueOf(fields.get(MapKey.STATUS.getName()).toString());
if (fields.get(MapKey.TEMPLATES.getName()) != null) {
this.templates = ((List<HashMap<String, Object>>) fields.get(MapKey.TEMPLATES.getName())).stream().map(hashMap -> new DatasetTempalate().fromElasticEntity(hashMap)).collect(Collectors.toList());
}
if (fields.get(MapKey.COLLABORATORS.getName()) != null) {
this.collaborators = ((List<HashMap<String, Object>>) fields.get(MapKey.COLLABORATORS.getName())).stream().map(map -> new Collaborator().fromElasticEntity(map)).collect(Collectors.toList());
}
if (fields.get(MapKey.ORGANIZATIONS.getName()) != null) {
this.organizations = ((List<HashMap<String, Object>>) fields.get(MapKey.ORGANIZATIONS.getName())).stream().map(map -> new Organization().fromElasticEntity(map)).collect(Collectors.toList());
}
this.lastVersion = (Boolean) fields.get(MapKey.LASTVERSION.getName());
this.lastPublicVersion = (Boolean) fields.get(MapKey.LASTPUBLICVERSION.getName());
this.isPublic = (Boolean) fields.get(MapKey.ISPUBLIC.getName());
if (fields.get(MapKey.DATASETS.getName()) != null) {
this.datasets = ((List<HashMap<String, Object>>) fields.get(MapKey.DATASETS.getName())).stream().map(map -> new Dataset().fromElasticEntity(map)).collect(Collectors.toList());
}
this.grant = UUID.fromString((String) fields.get(MapKey.GRANT.getName()));
if (fields.get(MapKey.GRANTSTATUS.getName()) != null) {
this.grantStatus = Short.valueOf(fields.get(MapKey.GRANTSTATUS.getName()).toString());
}
return this;
}
public enum MapKey {
ID ("id"),
LABEL ("label"),
DESCRIPTION ("description"),
GROUPID ("groupId"),
STATUS ("status"),
TEMPLATES ("templates"),
COLLABORATORS ("collaborators"),
ORGANIZATIONS ("organizations"),
LASTVERSION ("lastVersion"),
LASTPUBLICVERSION ("lastPublicVersion"),
ISPUBLIC ("isPublic"),
DATASETS ("datasets"),
GRANT ("grant"),
GRANTSTATUS ("grantStatus");
private final String name;
private MapKey(String name) {
this.name = name;
}
public String getName() {
return name;
}
}
}
......@@ -2,122 +2,174 @@ package eu.eudat.elastic.repository;
import eu.eudat.elastic.criteria.DatasetCriteria;
import eu.eudat.elastic.entities.Dataset;
import eu.eudat.elastic.entities.Dmp;
import eu.eudat.elastic.entities.Tag;
import org.elasticsearch.action.get.GetRequest;
import org.elasticsearch.action.get.GetResponse;
import org.apache.lucene.search.join.ScoreMode;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.client.core.CountRequest;
import org.elasticsearch.client.core.CountResponse;
import org.elasticsearch.client.indices.GetIndexRequest;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.InnerHitBuilder;
import org.elasticsearch.index.query.NestedQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.reindex.DeleteByQueryRequest;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.bucket.filter.FiltersAggregationBuilder;
import org.elasticsearch.search.aggregations.bucket.filter.ParsedFilters;
import org.elasticsearch.search.aggregations.bucket.nested.NestedAggregationBuilder;
import org.elasticsearch.search.aggregations.bucket.nested.ParsedNested;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.springframework.beans.factory.annotation.Autowired;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.UUID;
import java.util.*;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@Service("datasetRepository")
public class DatasetRepository extends ElasticRepository<Dataset, DatasetCriteria> {
@Autowired
public DatasetRepository(RestHighLevelClient client) {
private final DmpRepository dmpRepository;
public DatasetRepository(RestHighLevelClient client, DmpRepository dmpRepository) {
super(client);
this.dmpRepository = dmpRepository;
}
@Override
public Dataset createOrUpdate(Dataset entity) throws IOException {
if (this.getClient() != null) {
XContentBuilder builder = XContentFactory.jsonBuilder();
IndexRequest request = new IndexRequest("datasets").id(entity.getId()).source(entity.toElasticEntity(builder));//new IndexRequest("datasets", "doc", entity.getId()).source(entity.toElasticEntity(builder));
Dmp dmp = this.dmpRepository.findDocument(entity.getDmp().toString());
boolean found = false;
if (dmp.getDatasets() != null && !dmp.getDatasets().isEmpty()) {
for (int i = 0; i < dmp.getDatasets().size(); i++) {
if (dmp.getDatasets().get(i).getId().equals(entity.getId())) {
dmp.getDatasets().set(i, entity);
found = true;
break;
}
}
}
if (!found) {
if (dmp.getDatasets() == null) {
dmp.setDatasets(new ArrayList<>());
}
dmp.getDatasets().add(entity);
}
IndexRequest request = new IndexRequest("dmps").id(dmp.getId().toString()).source(dmp.toElasticEntity(builder));//new IndexRequest("datasets", "doc", entity.getId()).source(entity.toElasticEntity(builder));
this.getClient().index(request, RequestOptions.DEFAULT);
return entity;
}
return null;
}
@Override
public Dataset findDocument(String id) throws IOException {
GetRequest request = new GetRequest("datasets",id);
GetResponse response = this.getClient().get(request, RequestOptions.DEFAULT);
return new Dataset().fromElasticEntity(response.getSourceAsMap());
if (this.getClient() != null) {
SearchRequest searchRequest = new SearchRequest("dmps");
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery().should(QueryBuilders.termQuery("datasets.id.keyword", id));
NestedQueryBuilder nestedQueryBuilder = QueryBuilders.nestedQuery( "datasets", boolQuery, ScoreMode.Avg).innerHit(new InnerHitBuilder());
searchSourceBuilder.query(nestedQueryBuilder);
searchRequest.source(searchSourceBuilder);
SearchResponse response = this.getClient().search(searchRequest, RequestOptions.DEFAULT);
return ((Stream<Dataset>)Arrays.stream(response.getHits().getHits())
.map(hit -> hit.getInnerHits().values()).flatMap(Collection::stream)
.map(SearchHits::getHits).flatMap(Arrays::stream)
.map(x -> new Dataset().fromElasticEntity(this.transformFromString(x.getSourceAsString(), Map.class)))).findFirst().orElse(null);
// GetRequest request = new GetRequest("datasets", id);
// GetResponse response = this.getClient().get(request, RequestOptions.DEFAULT);
// return new Dataset().fromElasticEntity(response.getSourceAsMap());
}
return null;
}
@Override
public List<Dataset> query(DatasetCriteria criteria) throws IOException {
SearchRequest searchRequest = new SearchRequest("datasets");
if (this.getClient() != null) {
SearchRequest searchRequest = new SearchRequest("dmps");
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
CountRequest countRequest = new CountRequest("datasets");
countRequest.query(QueryBuilders.boolQuery().mustNot(QueryBuilders.termsQuery("status.keyword", Stream.of(Dataset.Status.DELETED.getValue(), Dataset.Status.CANCELED.getValue()).collect(Collectors.toList()))));
/*CountRequest countRequest = new CountRequest("dmps").routing("datasets").routing("id");
countRequest.query(QueryBuilders.boolQuery().mustNot(QueryBuilders.termsQuery("datasets.status.keyword", Stream.of(Dataset.Status.DELETED.getValue(), Dataset.Status.CANCELED.getValue()).collect(Collectors.toList()))));
CountResponse countResponse = getClient().count(countRequest, RequestOptions.DEFAULT);
Long count = countResponse.getCount();
Long count = countResponse.getCount();*/
SearchRequest countRequest = new SearchRequest("dmps");
NestedAggregationBuilder nestedAggregationBuilder = AggregationBuilders.nested("by_dataset", "datasets");
FiltersAggregationBuilder filtersAggregationBuilder = AggregationBuilders.filters("dataset_query", QueryBuilders.boolQuery().mustNot(QueryBuilders.termsQuery("datasets.status.keyword", Stream.of(Dataset.Status.DELETED.getValue(), Dataset.Status.CANCELED.getValue()).collect(Collectors.toList()))));
nestedAggregationBuilder.subAggregation(filtersAggregationBuilder);
SearchSourceBuilder countSourceBuilder = new SearchSourceBuilder();
countSourceBuilder.aggregation(nestedAggregationBuilder);
countRequest.source(countSourceBuilder);
SearchResponse countResponse = getClient().search(countRequest, RequestOptions.DEFAULT);
Long count = ((ParsedFilters)((ParsedNested)countResponse.getAggregations().asMap().get("by_dataset")).getAggregations().get("dataset_query")).getBuckets().get(0).getDocCount();
searchSourceBuilder.size(count.intValue());
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery().mustNot(QueryBuilders.termsQuery("status.keyword", Stream.of(Dataset.Status.DELETED.getValue(), Dataset.Status.CANCELED.getValue()).collect(Collectors.toList())));
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery().mustNot(QueryBuilders.termsQuery("datasets.status", Stream.of(Dataset.Status.DELETED.getValue(), Dataset.Status.CANCELED.getValue()).collect(Collectors.toList())));
if (criteria.isPublic()) {
boolQuery = boolQuery.should(QueryBuilders.termQuery("public.keyword", "true"));
boolQuery = boolQuery.should(QueryBuilders.termQuery("status.keyword", Dataset.Status.FINALISED.getValue()));
boolQuery = boolQuery.should(QueryBuilders.termQuery("lastPublicVersion.keyword", "true"));
boolQuery = boolQuery.should(QueryBuilders.termQuery("datasets.public", "true"));
boolQuery = boolQuery.should(QueryBuilders.termQuery("datasets.status", Dataset.Status.FINALISED.getValue()));
boolQuery = boolQuery.should(QueryBuilders.termQuery("datasets.lastPublicVersion", "true"));
}
if (criteria.getLike() != null && !criteria.getLike().isEmpty()) {
boolQuery = boolQuery.should(QueryBuilders.queryStringQuery(criteria.getLike()).fields(Stream.of(new Object[][] {
{ "label", 1.0f },
{ "description", 1.0f },
{ "formData", 1.0f }
boolQuery = boolQuery.should(QueryBuilders.queryStringQuery(criteria.getLike()).allowLeadingWildcard(true).fields(Stream.of(new Object[][]{
{"datasets.label", 1.0f},
{"datasets.description", 1.0f},
{"datasets.formData", 1.0f}
}).collect(Collectors.toMap(data -> (String) data[0], data -> (Float) data[1]))));
}
if (criteria.getDatasetTemplates() != null && criteria.getDatasetTemplates().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery("template.keyword", criteria.getDatasetTemplates().stream().map(UUID::toString).collect(Collectors.toList())));
boolQuery = boolQuery.should(QueryBuilders.termsQuery("datasets.template", criteria.getDatasetTemplates().stream().map(UUID::toString).collect(Collectors.toList())));
}
if (criteria.getStatus() != null) {
boolQuery = boolQuery.should(QueryBuilders.termQuery("status.keyword", criteria.getStatus().toString()));
boolQuery = boolQuery.should(QueryBuilders.termQuery("datasets.status", criteria.getStatus().toString()));
}
if (criteria.getDmps() != null && criteria.getDmps().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery("dmp.keyword", criteria.getDmps().stream().map(UUID::toString).collect(Collectors.toList())));
boolQuery = boolQuery.should(QueryBuilders.termsQuery("datasets.dmp", criteria.getDmps().stream().map(UUID::toString).collect(Collectors.toList())));
}
if (criteria.getGroupIds() != null && criteria.getGroupIds().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery("group.keyword", criteria.getGroupIds().stream().map(UUID::toString).collect(Collectors.toList())));
boolQuery = boolQuery.should(QueryBuilders.termsQuery("datasets.group", criteria.getGroupIds().stream().map(UUID::toString).collect(Collectors.toList())));
}
if (criteria.getGrants() != null && criteria.getGrants().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery("grant.keyword", criteria.getGrants().stream().map(UUID::toString).collect(Collectors.toList())));
boolQuery = boolQuery.should(QueryBuilders.termsQuery("datasets.grant", criteria.getGrants().stream().map(UUID::toString).collect(Collectors.toList())));
}
if (criteria.getGrantStatus() != null) {
boolQuery = boolQuery.should(QueryBuilders.termQuery("grantStatus.keyword", criteria.getGrantStatus().toString()));
boolQuery = boolQuery.should(QueryBuilders.termQuery("datasets.grantStatus", criteria.getGrantStatus().toString()));
}
if (criteria.getCollaborators() != null && criteria.getCollaborators().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery("collaborators.id.keyword", criteria.getCollaborators().stream().map(UUID::toString).collect(Collectors.toList())));
boolQuery = boolQuery.should(QueryBuilders.termsQuery("datasets.collaborators.id", criteria.getCollaborators().stream().map(UUID::toString).collect(Collectors.toList())));
}
if (!criteria.isPublic()) {
if (criteria.getAllowAllVersions() != null && !criteria.getAllowAllVersions()) {
boolQuery = boolQuery.should(QueryBuilders.termQuery("lastVersion.keyword", "true"));
boolQuery = boolQuery.should(QueryBuilders.termQuery("datasets.lastVersion", "true"));
}
}
if (criteria.getOrganiztions() != null && criteria.getOrganiztions().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery("organizations.id.keyword", criteria.getOrganiztions()));
boolQuery = boolQuery.should(QueryBuilders.termsQuery("datasets.organizations.id", criteria.getOrganiztions()));
}
if (criteria.getTags() != null && criteria.getTags().size() > 0) {
boolQuery = boolQuery.should(QueryBuilders.termsQuery("tags.name.keyword", criteria.getTags().stream().map(Tag::getName).collect(Collectors.toList())));
boolQuery = boolQuery.should(QueryBuilders.termsQuery("datasets.tags.name", criteria.getTags().stream().map(Tag::getName).collect(Collectors.toList())));
}
if (boolQuery.should().isEmpty() && boolQuery.mustNot().isEmpty()) {
......@@ -125,25 +177,36 @@ public class DatasetRepository extends ElasticRepository<Dataset, DatasetCriteri
} else {
boolQuery.minimumShouldMatch(boolQuery.should().size());
}
searchSourceBuilder.query(boolQuery);
NestedQueryBuilder nestedQueryBuilder = QueryBuilders.nestedQuery("datasets", boolQuery, ScoreMode.Avg).innerHit(new InnerHitBuilder());
searchSourceBuilder.query(nestedQueryBuilder);
searchRequest.source(searchSourceBuilder);
SearchResponse response = this.getClient().search(searchRequest, RequestOptions.DEFAULT);
return Arrays.stream(response.getHits().getHits()).map(x -> this.transformFromString(x.getSourceAsString(), Dataset.class)).collect(Collectors.toList());
return ((Stream<Dataset>)Arrays.stream(response.getHits().getHits())
.map(hit -> hit.getInnerHits().values()).flatMap(Collection::stream)
.map(SearchHits::getHits).flatMap(Arrays::stream)
.map(x -> new Dataset().fromElasticEntity(this.transformFromString(x.getSourceAsString(), Map.class)))).collect(Collectors.toList());
}
return null;
}
@Override
public boolean exists() throws IOException {
GetIndexRequest request = new GetIndexRequest("datasets");
if (this.getClient() != null) {
GetIndexRequest request = new GetIndexRequest("dmps");
// request.indices("datasets");
return this.getClient().indices().exists(request, RequestOptions.DEFAULT);
}
return false;
}
@Override
public void clear() throws IOException {
if (exists()) {
//DON'T
/* if (exists()) {
DeleteByQueryRequest delete = new DeleteByQueryRequest("datasets");
delete.setQuery(QueryBuilders.matchAllQuery());
this.getClient().deleteByQuery(delete, RequestOptions.DEFAULT);
}
}*/
}
}