ELK.Docker/shared/config-elk/logstash/pipeline/open_dmp_send_to_elastic.conf

@@ -12,7 +12,7 @@ output {
 		password =>
 		index =>"opendmp.logs"
 		#manage_template => true
-		#template => "/usr/share/logstash/templates/audit/cite_elas_openDMP.json"
+		#template => "/usr/share/logstash/templates/audit/openDMP.json"
 		#template_name => "cite.elas.openDMP-audit*" 
 		#template_overwrite => true
 	}

dmp-backend/Docker/dmp-backend.env

+PROFILE=staging
\ No newline at end of file

dmp-backend/Dockerfile

-FROM openjdk:8-jdk-alpine
-RUN apk add --update \
-        curl \
-        && rm -rf /var/cache/apk/*
-VOLUME /tmp
-ARG PROFILE=production
-ENV PROF $PROFILE
-ADD web/src/main/resources/ProjectConfiguration.xml /tmp/ProjectConfiguration.xml
-ADD web/src/main/resources/ExternalUrls.xml /tmp/ExternalUrls.xml
-ADD web/target/web-1.0-SNAPSHOT.jar app.jar
-ENTRYPOINT ["java","-Djava.security.egd=file:/dev/./urandom" ,"-Dspring.profiles.active=${PROF}","-jar","/app.jar"]
\ No newline at end of file
+FROM maven:3-jdk-8-alpine AS MAVEN_BUILD
+
+ 
+
+COPY pom.xml /build/
+COPY data /build/data/
+COPY elastic /build/elastic/
+COPY logging /build/logging/
+COPY queryable /build/queryable/
+COPY web /build/web/
+
+ 
+
+WORKDIR /build/
+RUN mvn package
+
+ 
+
+FROM openjdk:8-jre-alpine
+WORKDIR /app
+
+COPY --from=MAVEN_BUILD /build/web/target/web-1.0-SNAPSHOT.jar /app.jar
+ENTRYPOINT ["java","-Djava.security.egd=file:/dev/./urandom" ,"-Dspring.profiles.active=${PROFILE}","-jar","/app.jar"]
\ No newline at end of file

dmp-backend/data/src/main/java/eu/eudat/data/dao/criteria/DataManagementPlanCriteria.java

 package eu.eudat.data.dao.criteria;
 
 import eu.eudat.data.entities.DMP;
-
 import eu.eudat.data.entities.Grant;
 
 import java.util.Date;
@@ -20,6 +19,8 @@ public class DataManagementPlanCriteria extends Criteria<DMP> {
     private List<UUID> collaborators;
     private List<UUID> datasetTemplates;
     private boolean isPublic;
+    private boolean onlyPublic;
+    private Short grantStatus;
 
     public Date getPeriodStart() {
         return periodStart;
@@ -97,4 +98,20 @@ public class DataManagementPlanCriteria extends Criteria<DMP> {
     public void setIsPublic(boolean isPublic) {
         this.isPublic = isPublic;
     }
+
+    public boolean isOnlyPublic() {
+        return onlyPublic;
+    }
+
+    public void setOnlyPublic(boolean onlyPublic) {
+        this.onlyPublic = onlyPublic;
+    }
+
+    public Short getGrantStatus() {
+        return grantStatus;
+    }
+
+    public void setGrantStatus(Short grantStatus) {
+        this.grantStatus = grantStatus;
+    }
 }

dmp-backend/data/src/main/java/eu/eudat/data/dao/criteria/DatasetProfileCriteria.java

@@ -33,6 +33,7 @@ public class DatasetProfileCriteria extends Criteria<DatasetProfile> {
     private Short filter;
     private UUID userId;
     private boolean finalized;
+    private Integer status;
 
     public boolean getAllVersions() { return allVersions; }
     public void setAllVersions(boolean allVersions) { this.allVersions = allVersions; }
@@ -60,4 +61,12 @@ public class DatasetProfileCriteria extends Criteria<DatasetProfile> {
     public void setFinalized(boolean finalized) {
         this.finalized = finalized;
     }
+
+    public Integer getStatus() {
+        return status;
+    }
+
+    public void setStatus(Integer status) {
+        this.status = status;
+    }
 }

dmp-backend/data/src/main/java/eu/eudat/data/dao/entities/DMPDaoImpl.java

@@ -9,6 +9,7 @@ import eu.eudat.data.entities.UserInfo;
 import eu.eudat.queryable.QueryableList;
 import eu.eudat.queryable.types.FieldSelectionType;
 import eu.eudat.queryable.types.SelectionField;
+import eu.eudat.types.grant.GrantStateType;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.scheduling.annotation.Async;
 import org.springframework.stereotype.Component;
@@ -16,6 +17,7 @@ import org.springframework.stereotype.Component;
 import javax.persistence.criteria.Join;
 import javax.persistence.criteria.JoinType;
 import java.util.Arrays;
+import java.util.Date;
 import java.util.List;
 import java.util.UUID;
 import java.util.concurrent.CompletableFuture;
@@ -73,6 +75,14 @@ public class DMPDaoImpl extends DatabaseAccess<DMP> implements DMPDao {
 		if (criteria.getDatasetTemplates() != null && !criteria.getDatasetTemplates().isEmpty()) {
 			query.where((builder, root) -> root.join("associatedDmps", JoinType.LEFT).get("id").in(criteria.getDatasetTemplates()));
 		}
+		if (criteria.getGrantStatus() != null) {
+			if (criteria.getGrantStatus().equals(GrantStateType.FINISHED.getValue().shortValue()))
+				query.where((builder, root) -> builder.lessThan(root.get("grant").get("enddate"), new Date()));
+			if (criteria.getGrantStatus().equals(GrantStateType.ONGOING.getValue().shortValue()))
+				query.where((builder, root) ->
+						builder.or(builder.greaterThan(root.get("grant").get("enddate"), new Date())
+								, builder.isNull(root.get("grant").get("enddate"))));
+		}
 		query.where((builder, root) -> builder.notEqual(root.get("status"), DMP.DMPStatus.DELETED.getValue()));
 		return query;
 	}

dmp-backend/data/src/main/java/eu/eudat/data/dao/entities/DatasetProfileDaoImpl.java

@@ -52,6 +52,9 @@ public class DatasetProfileDaoImpl extends DatabaseAccess<DatasetProfile> implem
                                 builder.notEqual(root.get("id"), criteria.getUserId())));
             }
         }
+        if (criteria.getStatus() != null) {
+            query.where(((builder, root) -> builder.equal(root.get("status"), criteria.getStatus())));
+        }
         if (criteria.getFinalized()) {
             query.where(((builder, root) -> builder.equal(root.get("status"), DatasetProfile.Status.FINALIZED.getValue())));
         } else {

dmp-backend/data/src/main/java/eu/eudat/data/dao/entities/DoiFunderDao.java

+package eu.eudat.data.dao.entities;
+
+import eu.eudat.data.dao.DatabaseAccessLayer;
+import eu.eudat.data.entities.DoiFunder;
+
+import java.util.UUID;
+
+public interface DoiFunderDao extends DatabaseAccessLayer<DoiFunder, UUID> {
+
+	DoiFunder findFunderByName(String name);
+}

dmp-backend/data/src/main/java/eu/eudat/data/dao/entities/DoiFunderDaoImpl.java

+package eu.eudat.data.dao.entities;
+
+import eu.eudat.data.dao.DatabaseAccess;
+import eu.eudat.data.dao.databaselayer.service.DatabaseService;
+import eu.eudat.data.entities.DoiFunder;
+import eu.eudat.queryable.QueryableList;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.util.UUID;
+import java.util.concurrent.CompletableFuture;
+
+@Component("DoiFunderDao")
+public class DoiFunderDaoImpl extends DatabaseAccess<DoiFunder> implements DoiFunderDao {
+
+	@Autowired
+	public DoiFunderDaoImpl(DatabaseService<DoiFunder> databaseService) {
+		super(databaseService);
+	}
+
+	@Override
+	public DoiFunder findFunderByName(String name) {
+		return this.asQueryable().toList().stream().filter(doiFunder -> name.contains(doiFunder.getName()) || doiFunder.getName().contains(name)).findFirst().orElse(null);
+	}
+
+	@Override
+	public DoiFunder createOrUpdate(DoiFunder item) {
+		return this.getDatabaseService().createOrUpdate(item, DoiFunder.class);
+	}
+
+	@Override
+	public CompletableFuture<DoiFunder> createOrUpdateAsync(DoiFunder item) {
+		return CompletableFuture.supplyAsync(() -> this.createOrUpdate(item));
+	}
+
+	@Override
+	public DoiFunder find(UUID id) {
+		return this.getDatabaseService().getQueryable(DoiFunder.class).where(((builder, root) -> builder.equal(root.get("id"), id))).getSingle();
+	}
+
+	@Override
+	public DoiFunder find(UUID id, String hint) {
+		throw new UnsupportedOperationException();
+	}
+
+	@Override
+	public void delete(DoiFunder item) {
+		this.getDatabaseService().delete(item);
+	}
+
+	@Override
+	public QueryableList<DoiFunder> asQueryable() {
+		return this.getDatabaseService().getQueryable(DoiFunder.class);
+	}
+}

dmp-backend/data/src/main/java/eu/eudat/data/dao/entities/UserAssociationDao.java

+package eu.eudat.data.dao.entities;
+
+import eu.eudat.data.dao.DatabaseAccessLayer;
+import eu.eudat.data.entities.UserAssociation;
+import eu.eudat.data.entities.UserInfo;
+
+import java.util.List;
+import java.util.UUID;
+
+public interface UserAssociationDao extends DatabaseAccessLayer<UserAssociation, UUID> {
+
+	public List<UserAssociation> getAssociated(UserInfo userId);
+
+	public Boolean areAssociated(UserInfo firstUser, UserInfo secondUser);
+}

dmp-backend/data/src/main/java/eu/eudat/data/dao/entities/UserAssociationDaoImpl.java

+package eu.eudat.data.dao.entities;
+
+import eu.eudat.data.dao.DatabaseAccess;
+import eu.eudat.data.dao.databaselayer.service.DatabaseService;
+import eu.eudat.data.entities.UserAssociation;
+import eu.eudat.data.entities.UserInfo;
+import eu.eudat.queryable.QueryableList;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.CompletableFuture;
+
+@Component("UserAssociationDao")
+public class UserAssociationDaoImpl extends DatabaseAccess<UserAssociation> implements UserAssociationDao {
+
+	@Autowired
+	public UserAssociationDaoImpl(DatabaseService<UserAssociation> databaseService) {
+		super(databaseService);
+	}
+
+	@Override
+	public UserAssociation createOrUpdate(UserAssociation item) {
+		return this.getDatabaseService().createOrUpdate(item, UserAssociation.class);
+	}
+
+	@Override
+	public CompletableFuture<UserAssociation> createOrUpdateAsync(UserAssociation item) {
+		return CompletableFuture.supplyAsync(() -> this.createOrUpdate(item));
+	}
+
+	@Override
+	public UserAssociation find(UUID id) {
+		return this.getDatabaseService().getQueryable(UserAssociation.class).where(((builder, root) -> builder.equal(root.get("id"), id))).getSingle();
+	}
+
+	@Override
+	public UserAssociation find(UUID id, String hint) {
+		throw new UnsupportedOperationException();
+	}
+
+	@Override
+	public void delete(UserAssociation item) {
+		this.getDatabaseService().delete(item);
+	}
+
+	@Override
+	public QueryableList<UserAssociation> asQueryable() {
+		return this.getDatabaseService().getQueryable(UserAssociation.class);
+	}
+
+	@Override
+	public List<UserAssociation> getAssociated(UserInfo userId) {
+		return this.getDatabaseService().getQueryable(UserAssociation.class).where(((builder, root) ->
+			builder.or(builder.equal(root.get("firstUser"), userId), builder.equal(root.get("secondUser"), userId)))).toList();
+	}
+
+	@Override
+	public Boolean areAssociated(UserInfo firstUser, UserInfo secondUser) {
+		return this.getDatabaseService().getQueryable(UserAssociation.class).where(((builder, root) ->
+				builder.or(
+						builder.and(
+								builder.equal(root.get("firstUser"), firstUser),
+								builder.equal(root.get("secondUser"), secondUser)
+						),
+						builder.and(
+								builder.equal(root.get("secondUser"), firstUser),
+								builder.equal(root.get("firstUser"), secondUser)
+						)
+				))).count() > 0;
+	}
+}

dmp-backend/data/src/main/java/eu/eudat/data/entities/DataRepository.java

@@ -132,6 +132,9 @@ public class DataRepository implements Serializable, DataEntity<DataRepository,
 
 	@Override
 	public void update(DataRepository entity) {
+		this.label = entity.getLabel();
+		this.abbreviation = entity.getAbbreviation();
+		this.uri = entity.getUri();
 
 	}
 

dmp-backend/data/src/main/java/eu/eudat/data/entities/DoiFunder.java

+package eu.eudat.data.entities;
+
+import eu.eudat.queryable.queryableentity.DataEntity;
+
+import javax.persistence.*;
+import java.util.List;
+import java.util.UUID;
+
+@Entity
+@Table(name = "\"DoiFunder\"")
+public class DoiFunder implements DataEntity<DoiFunder, UUID> {
+
+	@Id
+	private UUID id;
+
+	@Column(name = "name")
+	private String name;
+
+	@Column(name = "doi")
+	private String doi;
+
+	public UUID getId() {
+		return id;
+	}
+
+	public void setId(UUID id) {
+		this.id = id;
+	}
+
+	public String getName() {
+		return name;
+	}
+
+	public void setName(String name) {
+		this.name = name;
+	}
+
+	public String getDoi() {
+		return doi;
+	}
+
+	public void setDoi(String doi) {
+		this.doi = doi;
+	}
+
+	@Override
+	public void update(DoiFunder entity) {
+		this.name = entity.name;
+		this.doi = entity.doi;
+	}
+
+	@Override
+	public UUID getKeys() {
+		return id;
+	}
+
+	@Override
+	public DoiFunder buildFromTuple(List<Tuple> tuple, List<String> fields, String base) {
+		return null;
+	}
+}

dmp-backend/data/src/main/java/eu/eudat/data/entities/Registry.java

@@ -139,7 +139,9 @@ public class Registry implements DataEntity<Registry, UUID> {
 
 	@Override
 	public void update(Registry entity) {
-
+		this.label = entity.getLabel();
+		this.abbreviation = entity.getAbbreviation();
+		this.uri = entity.getUri();
 	}
 
 	@Override

dmp-backend/data/src/main/java/eu/eudat/data/entities/Service.java

@@ -134,7 +134,9 @@ public class Service implements DataEntity<Service, UUID> {
 
 	@Override
 	public void update(Service entity) {
-
+		this.label = entity.getLabel();
+		this.abbreviation = entity.getAbbreviation();
+		this.uri = entity.getUri();
 	}
 
 	@Override

dmp-backend/data/src/main/java/eu/eudat/data/entities/UserAssociation.java

+package eu.eudat.data.entities;
+
+import eu.eudat.queryable.queryableentity.DataEntity;
+import org.hibernate.annotations.GenericGenerator;
+
+import javax.persistence.*;
+import java.util.List;
+import java.util.UUID;
+
+@Entity
+@Table(name = "\"UserAssociation\"")
+public class UserAssociation implements DataEntity<UserAssociation, UUID> {
+
+	@Id
+	@GeneratedValue
+	@GenericGenerator(name = "uuid2", strategy = "uuid2")
+	@Column(name = "id", updatable = false, nullable = false, columnDefinition = "BINARY(16)")
+	private UUID id;
+
+	@OneToOne(fetch = FetchType.LAZY)
+	@JoinColumn(name = "\"firstUser\"")
+	private UserInfo firstUser;
+
+	@OneToOne(fetch = FetchType.LAZY)
+	@JoinColumn(name = "\"secondUser\"")
+	private UserInfo secondUser;
+
+	public UUID getId() {
+		return id;
+	}
+
+	public void setId(UUID id) {
+		this.id = id;
+	}
+
+	public UserInfo getFirstUser() {
+		return firstUser;
+	}
+
+	public void setFirstUser(UserInfo firstUser) {
+		this.firstUser = firstUser;
+	}
+
+	public UserInfo getSecondUser() {
+		return secondUser;
+	}
+
+	public void setSecondUser(UserInfo secondUser) {
+		this.secondUser = secondUser;
+	}
+
+	@Override
+	public void update(UserAssociation entity) {
+
+	}
+
+	@Override
+	public UUID getKeys() {
+		return null;
+	}
+
+	@Override
+	public UserAssociation buildFromTuple(List<Tuple> tuple, List<String> fields, String base) {
+		return null;
+	}
+}

dmp-backend/elastic/src/main/java/eu/eudat/elastic/criteria/DmpCriteria.java

+package eu.eudat.elastic.criteria;
+
+import java.util.List;
+import java.util.UUID;
+
+public class DmpCriteria extends Criteria {
+	private String like;
+	private Short status;
+	private List<UUID> templates;
+	private List<UUID> grants;
+	private List<UUID> collaborators;
+	private List<UUID> organizations;
+	private boolean isPublic;
+	private List<UUID> groupIds;
+	private boolean allowAllVersions;
+	private Short grantStatus;
+
+	public String getLike() {
+		return like;
+	}
+
+	public void setLike(String like) {
+		this.like = like;
+	}
+
+	public Short getStatus() {
+		return status;
+	}
+
+	public void setStatus(Short status) {
+		this.status = status;
+	}
+
+	public List<UUID> getTemplates() {
+		return templates;
+	}
+
+	public void setTemplates(List<UUID> templates) {
+		this.templates = templates;
+	}
+
+	public List<UUID> getGrants() {
+		return grants;
+	}
+
+	public void setGrants(List<UUID> grants) {
+		this.grants = grants;
+	}
+
+	public List<UUID> getCollaborators() {
+		return collaborators;
+	}
+
+	public void setCollaborators(List<UUID> collaborators) {
+		this.collaborators = collaborators;
+	}
+
+	public List<UUID> getOrganizations() {
+		return organizations;
+	}
+
+	public void setOrganizations(List<UUID> organizations) {
+		this.organizations = organizations;
+	}
+
+	public boolean isPublic() {
+		return isPublic;
+	}
+
+	public void setPublic(boolean aPublic) {
+		isPublic = aPublic;
+	}
+
+	public List<UUID> getGroupIds() {
+		return groupIds;
+	}
+
+	public void setGroupIds(List<UUID> groupIds) {
+		this.groupIds = groupIds;
+	}
+
+	public boolean isAllowAllVersions() {
+		return allowAllVersions;
+	}
+
+	public void setAllowAllVersions(boolean allowAllVersions) {
+		this.allowAllVersions = allowAllVersions;
+	}
+
+	public Short getGrantStatus() {
+		return grantStatus;
+	}
+
+	public void setGrantStatus(Short grantStatus) {
+		this.grantStatus = grantStatus;
+	}
+}

dmp-backend/elastic/src/main/java/eu/eudat/elastic/entities/DatasetTempalate.java

+package eu.eudat.elastic.entities;
+
+import org.elasticsearch.common.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.UUID;
+
+public class DatasetTempalate implements ElasticEntity<DatasetTempalate> {
+	private UUID id;
+	private String name;
+
+	public UUID getId() {
+		return id;
+	}
+
+	public void setId(UUID id) {
+		this.id = id;
+	}
+
+	public String getName() {
+		return name;
+	}
+
+	public void setName(String name) {
+		this.name = name;
+	}
+
+	@Override
+	public XContentBuilder toElasticEntity(XContentBuilder builder) throws IOException {
+		builder.startObject();
+		builder.field("id", this.id.toString());
+		builder.field("name", this.name);
+		builder.endObject();
+		return builder;
+	}
+
+	@Override
+	public DatasetTempalate fromElasticEntity(Map<String, Object> fields) {
+		this.id = UUID.fromString((String) fields.get("id"));
+		this.name = (String) fields.get("name");
+		return this;
+	}
+}

dmp-backend/elastic/src/main/java/eu/eudat/elastic/entities/Dmp.java

+package eu.eudat.elastic.entities;
+
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.stream.Collectors;
+
+public class Dmp implements ElasticEntity<Dmp> {
+	private static final Logger logger = LoggerFactory.getLogger(Dmp.class);
+
+	public enum DMPStatus {
+		ACTIVE((short) 0), FINALISED((short) 1),DELETED((short) 99);
+
+		private short value;
+
+		private DMPStatus(short value) {
+			this.value = value;
+		}
+
+		public short getValue() {
+			return value;
+		}
+
+		public static DMPStatus fromInteger(short value) {
+			switch (value) {
+				case 0:
+					return ACTIVE;
+				case 1:
+					return FINALISED;
+				case 99:
+					return DELETED;
+				default:
+					throw new RuntimeException("Unsupported DMP Status");
+			}
+		}
+	}
+
+	private UUID id;
+	private String label;
+	private String description;
+	private UUID groupId;
+	private Short status;
+	private List<DatasetTempalate> templates;
+	private List<Collaborator> collaborators;
+	private List<Organization> organizations;
+	private Boolean lastVersion;
+	private Boolean lastPublicVersion;
+	private Boolean isPublic;
+	private List<Dataset> datasets;
+	private UUID grant;
+	private Short grantStatus;
+
+	public UUID getId() {
+		return id;
+	}
+
+	public void setId(UUID id) {
+		this.id = id;
+	}
+
+	public String getLabel() {
+		return label;
+	}
+
+	public void setLabel(String label) {
+		this.label = label;
+	}
+
+	public String getDescription() {
+		return description;
+	}
+
+	public void setDescription(String description) {
+		this.description = description;
+	}
+
+	public UUID getGroupId() {
+		return groupId;
+	}
+
+	public void setGroupId(UUID groupId) {
+		this.groupId = groupId;
+	}
+
+	public Short getStatus() {
+		return status;
+	}
+
+	public void setStatus(Short status) {
+		this.status = status;
+	}
+
+	public List<DatasetTempalate> getTemplates() {
+		return templates;
+	}
+
+	public void setTemplates(List<DatasetTempalate> templates) {
+		this.templates = templates;
+	}
+
+	public List<Collaborator> getCollaborators() {
+		return collaborators;
+	}
+
+	public void setCollaborators(List<Collaborator> collaborators) {
+		this.collaborators = collaborators;
+	}
+
+	public List<Organization> getOrganizations() {
+		return organizations;
+	}
+
+	public void setOrganizations(List<Organization> organizations) {
+		this.organizations = organizations;
+	}
+
+	public Boolean getLastVersion() {
+		return lastVersion;
+	}
+
+	public void setLastVersion(Boolean lastVersion) {
+		this.lastVersion = lastVersion;
+	}
+
+	public Boolean getLastPublicVersion() {
+		return lastPublicVersion;
+	}
+
+	public void setLastPublicVersion(Boolean lastPublicVersion) {
+		this.lastPublicVersion = lastPublicVersion;
+	}
+
+	public Boolean getPublic() {
+		return isPublic;
+	}
+
+	public void setPublic(Boolean aPublic) {
+		isPublic = aPublic;
+	}
+
+	public List<Dataset> getDatasets() {
+		return datasets;
+	}
+
+	public void setDatasets(List<Dataset> datasets) {
+		this.datasets = datasets;
+	}
+
+	public UUID getGrant() {
+		return grant;
+	}
+
+	public void setGrant(UUID grant) {
+		this.grant = grant;
+	}
+
+	public Short getGrantStatus() {
+		return grantStatus;
+	}
+
+	public void setGrantStatus(Short grantStatus) {
+		this.grantStatus = grantStatus;
+	}
+
+	@Override
+	public XContentBuilder toElasticEntity(XContentBuilder builder) throws IOException {
+		builder.startObject();
+		if (this.id != null) {
+			builder.field(MapKey.ID.getName(), this.id.toString());
+		}
+		builder.field(MapKey.LABEL.getName(), this.label);
+		builder.field(MapKey.DESCRIPTION.getName(), this.description);
+		if (this.groupId != null) {
+			builder.field(MapKey.GROUPID.getName(), this.groupId.toString());
+		}
+		builder.field(MapKey.STATUS.getName(), this.status);
+		if (this.templates != null && !this.templates.isEmpty()) {
+			builder.startArray(MapKey.TEMPLATES.getName());
+			this.templates.forEach(template -> {
+				try {
+					template.toElasticEntity(builder);
+				} catch (IOException e) {
+					logger.error(e.getMessage(), e);
+				}
+			});
+			builder.endArray();
+		}
+		if (this.collaborators != null && !this.collaborators.isEmpty()) {
+			builder.startArray(MapKey.COLLABORATORS.getName());
+			this.collaborators.forEach(collaborator -> {
+				try {
+					collaborator.toElasticEntity(builder);
+				} catch (IOException e) {
+					logger.error(e.getMessage(), e);
+				}
+			});
+			builder.endArray();
+		}
+		if (this.organizations != null && !this.organizations.isEmpty()) {
+			builder.startArray(MapKey.ORGANIZATIONS.getName());
+			this.organizations.forEach(organization -> {
+				try {
+					organization.toElasticEntity(builder);
+				} catch (IOException e) {
+					logger.error(e.getMessage(), e);
+				}
+			});
+			builder.endArray();
+		}
+		builder.field(MapKey.LASTVERSION.getName(), this.lastVersion);
+		builder.field(MapKey.LASTPUBLICVERSION.getName(), this.lastPublicVersion);
+		builder.field(MapKey.ISPUBLIC.getName(), this.isPublic);
+		if (datasets != null && !this.datasets.isEmpty()) {
+			builder.startArray(MapKey.DATASETS.getName());
+			this.datasets.forEach(dataset -> {
+				try {
+					if (dataset != null) {
+						dataset.toElasticEntity(builder);
+					}
+				} catch (Exception e) {
+					logger.error(e.getMessage(), e);
+				}
+			});
+			builder.endArray();
+		}
+		if (this.grant != null) {
+			builder.field(MapKey.GRANT.getName(), this.grant.toString());
+		}
+		builder.field(MapKey.GRANTSTATUS.getName(), this.grantStatus);
+		builder.endObject();
+		return builder;
+	}
+
+	@Override
+	public Dmp fromElasticEntity(Map<String, Object> fields) {
+		this.id = UUID.fromString((String) fields.get(MapKey.ID.getName()));
+		this.label = (String) fields.get(MapKey.LABEL.getName());
+		this.description = (String) fields.get(MapKey.DESCRIPTION.getName());
+		if (fields.get(MapKey.GROUPID.getName()) != null) {
+			this.groupId = UUID.fromString((String) fields.get(MapKey.GROUPID.getName()));
+		}
+		this.status = Short.valueOf(fields.get(MapKey.STATUS.getName()).toString());
+		if (fields.get(MapKey.TEMPLATES.getName()) != null) {
+			this.templates = ((List<HashMap<String, Object>>) fields.get(MapKey.TEMPLATES.getName())).stream().map(hashMap -> new DatasetTempalate().fromElasticEntity(hashMap)).collect(Collectors.toList());
+		}
+		if (fields.get(MapKey.COLLABORATORS.getName()) != null) {
+			this.collaborators = ((List<HashMap<String, Object>>) fields.get(MapKey.COLLABORATORS.getName())).stream().map(map -> new Collaborator().fromElasticEntity(map)).collect(Collectors.toList());
+		}
+		if (fields.get(MapKey.ORGANIZATIONS.getName()) != null) {
+			this.organizations = ((List<HashMap<String, Object>>) fields.get(MapKey.ORGANIZATIONS.getName())).stream().map(map -> new Organization().fromElasticEntity(map)).collect(Collectors.toList());
+		}
+		this.lastVersion = (Boolean) fields.get(MapKey.LASTVERSION.getName());
+		this.lastPublicVersion = (Boolean) fields.get(MapKey.LASTPUBLICVERSION.getName());
+		this.isPublic = (Boolean) fields.get(MapKey.ISPUBLIC.getName());
+		if (fields.get(MapKey.DATASETS.getName()) != null) {
+			this.datasets = ((List<HashMap<String, Object>>) fields.get(MapKey.DATASETS.getName())).stream().map(map -> new Dataset().fromElasticEntity(map)).collect(Collectors.toList());
+		}
+		this.grant = UUID.fromString((String) fields.get(MapKey.GRANT.getName()));
+		if (fields.get(MapKey.GRANTSTATUS.getName()) != null) {
+			this.grantStatus = Short.valueOf(fields.get(MapKey.GRANTSTATUS.getName()).toString());
+		}
+		return this;
+	}
+
+	public enum MapKey {
+		ID ("id"),
+		LABEL ("label"),
+		DESCRIPTION ("description"),
+		GROUPID ("groupId"),
+		STATUS ("status"),
+		TEMPLATES ("templates"),
+		COLLABORATORS ("collaborators"),
+		ORGANIZATIONS ("organizations"),
+		LASTVERSION ("lastVersion"),
+		LASTPUBLICVERSION ("lastPublicVersion"),
+		ISPUBLIC ("isPublic"),
+		DATASETS ("datasets"),
+		GRANT ("grant"),
+		GRANTSTATUS ("grantStatus");
+
+		private final String name;
+
+		private MapKey(String name) {
+			this.name = name;
+		}
+
+		public String getName() {
+			return name;
+		}
+	}
+}

dmp-backend/elastic/src/main/java/eu/eudat/elastic/repository/DatasetRepository.java

@@ -2,122 +2,174 @@ package eu.eudat.elastic.repository;
 
 import eu.eudat.elastic.criteria.DatasetCriteria;
 import eu.eudat.elastic.entities.Dataset;
+import eu.eudat.elastic.entities.Dmp;
 import eu.eudat.elastic.entities.Tag;
-import org.elasticsearch.action.get.GetRequest;
-import org.elasticsearch.action.get.GetResponse;
+import org.apache.lucene.search.join.ScoreMode;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.RestHighLevelClient;
-import org.elasticsearch.client.core.CountRequest;
-import org.elasticsearch.client.core.CountResponse;
 import org.elasticsearch.client.indices.GetIndexRequest;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.InnerHitBuilder;
+import org.elasticsearch.index.query.NestedQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.index.reindex.DeleteByQueryRequest;
+import org.elasticsearch.search.SearchHits;
+import org.elasticsearch.search.aggregations.AggregationBuilders;
+import org.elasticsearch.search.aggregations.bucket.filter.FiltersAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.filter.ParsedFilters;
+import org.elasticsearch.search.aggregations.bucket.nested.NestedAggregationBuilder;
+import org.elasticsearch.search.aggregations.bucket.nested.ParsedNested;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
-import org.springframework.beans.factory.annotation.Autowired;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.springframework.stereotype.Service;
 
 import java.io.IOException;
-import java.util.Arrays;
-import java.util.List;
-import java.util.UUID;
+import java.util.*;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 @Service("datasetRepository")
 public class DatasetRepository extends ElasticRepository<Dataset, DatasetCriteria> {
 
-    @Autowired
-    public DatasetRepository(RestHighLevelClient client) {
+    private final DmpRepository dmpRepository;
+
+    public DatasetRepository(RestHighLevelClient client, DmpRepository dmpRepository) {
         super(client);
+        this.dmpRepository = dmpRepository;
     }
 
     @Override
     public Dataset createOrUpdate(Dataset entity) throws IOException {
+        if (this.getClient() != null) {
             XContentBuilder builder = XContentFactory.jsonBuilder();
-        IndexRequest request = new IndexRequest("datasets").id(entity.getId()).source(entity.toElasticEntity(builder));//new IndexRequest("datasets", "doc", entity.getId()).source(entity.toElasticEntity(builder));
+            Dmp dmp = this.dmpRepository.findDocument(entity.getDmp().toString());
+            boolean found = false;
+            if (dmp.getDatasets() != null && !dmp.getDatasets().isEmpty()) {
+                for (int i = 0; i < dmp.getDatasets().size(); i++) {
+                    if (dmp.getDatasets().get(i).getId().equals(entity.getId())) {
+                        dmp.getDatasets().set(i, entity);
+                        found = true;
+                        break;
+                    }
+                }
+            }
+            if (!found) {
+                if (dmp.getDatasets() == null) {
+                    dmp.setDatasets(new ArrayList<>());
+                }
+                dmp.getDatasets().add(entity);
+            }
+            IndexRequest request = new IndexRequest("dmps").id(dmp.getId().toString()).source(dmp.toElasticEntity(builder));//new IndexRequest("datasets", "doc", entity.getId()).source(entity.toElasticEntity(builder));
             this.getClient().index(request, RequestOptions.DEFAULT);
             return entity;
         }
+        return null;
+    }
 
     @Override
     public Dataset findDocument(String id) throws IOException {
-        GetRequest request = new GetRequest("datasets",id);
-        GetResponse response = this.getClient().get(request, RequestOptions.DEFAULT);
-        return new Dataset().fromElasticEntity(response.getSourceAsMap());
+        if (this.getClient() != null) {
+            SearchRequest searchRequest = new SearchRequest("dmps");
+            SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+            BoolQueryBuilder boolQuery = QueryBuilders.boolQuery().should(QueryBuilders.termQuery("datasets.id.keyword", id));
+            NestedQueryBuilder nestedQueryBuilder = QueryBuilders.nestedQuery( "datasets", boolQuery, ScoreMode.Avg).innerHit(new InnerHitBuilder());
+            searchSourceBuilder.query(nestedQueryBuilder);
+            searchRequest.source(searchSourceBuilder);
+            SearchResponse response = this.getClient().search(searchRequest, RequestOptions.DEFAULT);
+            return ((Stream<Dataset>)Arrays.stream(response.getHits().getHits())
+                                           .map(hit -> hit.getInnerHits().values()).flatMap(Collection::stream)
+                                           .map(SearchHits::getHits).flatMap(Arrays::stream)
+                                           .map(x -> new Dataset().fromElasticEntity(this.transformFromString(x.getSourceAsString(), Map.class)))).findFirst().orElse(null);
+//            GetRequest request = new GetRequest("datasets", id);
+//            GetResponse response = this.getClient().get(request, RequestOptions.DEFAULT);
+//            return new Dataset().fromElasticEntity(response.getSourceAsMap());
+        }
+        return null;
     }
 
     @Override
     public List<Dataset> query(DatasetCriteria criteria) throws IOException {
-        SearchRequest searchRequest = new SearchRequest("datasets");
+        if (this.getClient() != null) {
+            SearchRequest searchRequest = new SearchRequest("dmps");
             SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
 
-        CountRequest countRequest = new CountRequest("datasets");
-        countRequest.query(QueryBuilders.boolQuery().mustNot(QueryBuilders.termsQuery("status.keyword", Stream.of(Dataset.Status.DELETED.getValue(), Dataset.Status.CANCELED.getValue()).collect(Collectors.toList()))));
+            /*CountRequest countRequest = new CountRequest("dmps").routing("datasets").routing("id");
+            countRequest.query(QueryBuilders.boolQuery().mustNot(QueryBuilders.termsQuery("datasets.status.keyword", Stream.of(Dataset.Status.DELETED.getValue(), Dataset.Status.CANCELED.getValue()).collect(Collectors.toList()))));
             CountResponse countResponse = getClient().count(countRequest, RequestOptions.DEFAULT);
-        Long count = countResponse.getCount();
+            Long count = countResponse.getCount();*/
+
+            SearchRequest countRequest = new SearchRequest("dmps");
+            NestedAggregationBuilder nestedAggregationBuilder = AggregationBuilders.nested("by_dataset", "datasets");
+            FiltersAggregationBuilder filtersAggregationBuilder = AggregationBuilders.filters("dataset_query", QueryBuilders.boolQuery().mustNot(QueryBuilders.termsQuery("datasets.status.keyword", Stream.of(Dataset.Status.DELETED.getValue(), Dataset.Status.CANCELED.getValue()).collect(Collectors.toList()))));
+            nestedAggregationBuilder.subAggregation(filtersAggregationBuilder);
+            SearchSourceBuilder countSourceBuilder = new SearchSourceBuilder();
+            countSourceBuilder.aggregation(nestedAggregationBuilder);
+            countRequest.source(countSourceBuilder);
+            SearchResponse countResponse = getClient().search(countRequest, RequestOptions.DEFAULT);
+            Long count = ((ParsedFilters)((ParsedNested)countResponse.getAggregations().asMap().get("by_dataset")).getAggregations().get("dataset_query")).getBuckets().get(0).getDocCount();
+
 
             searchSourceBuilder.size(count.intValue());
 
-        BoolQueryBuilder boolQuery = QueryBuilders.boolQuery().mustNot(QueryBuilders.termsQuery("status.keyword", Stream.of(Dataset.Status.DELETED.getValue(), Dataset.Status.CANCELED.getValue()).collect(Collectors.toList())));
+            BoolQueryBuilder boolQuery = QueryBuilders.boolQuery().mustNot(QueryBuilders.termsQuery("datasets.status", Stream.of(Dataset.Status.DELETED.getValue(), Dataset.Status.CANCELED.getValue()).collect(Collectors.toList())));
             if (criteria.isPublic()) {
-            boolQuery = boolQuery.should(QueryBuilders.termQuery("public.keyword", "true"));
-            boolQuery = boolQuery.should(QueryBuilders.termQuery("status.keyword", Dataset.Status.FINALISED.getValue()));
-            boolQuery = boolQuery.should(QueryBuilders.termQuery("lastPublicVersion.keyword", "true"));
+                boolQuery = boolQuery.should(QueryBuilders.termQuery("datasets.public", "true"));
+                boolQuery = boolQuery.should(QueryBuilders.termQuery("datasets.status", Dataset.Status.FINALISED.getValue()));
+                boolQuery = boolQuery.should(QueryBuilders.termQuery("datasets.lastPublicVersion", "true"));
             }
             if (criteria.getLike() != null && !criteria.getLike().isEmpty()) {
-            boolQuery = boolQuery.should(QueryBuilders.queryStringQuery(criteria.getLike()).fields(Stream.of(new Object[][] {
-                    { "label", 1.0f },
-                    { "description", 1.0f },
-                    { "formData", 1.0f }
+                boolQuery = boolQuery.should(QueryBuilders.queryStringQuery(criteria.getLike()).allowLeadingWildcard(true).fields(Stream.of(new Object[][]{
+                        {"datasets.label", 1.0f},
+                        {"datasets.description", 1.0f},
+                        {"datasets.formData", 1.0f}
                 }).collect(Collectors.toMap(data -> (String) data[0], data -> (Float) data[1]))));
             }
 
             if (criteria.getDatasetTemplates() != null && criteria.getDatasetTemplates().size() > 0) {
-            boolQuery = boolQuery.should(QueryBuilders.termsQuery("template.keyword", criteria.getDatasetTemplates().stream().map(UUID::toString).collect(Collectors.toList())));
+                boolQuery = boolQuery.should(QueryBuilders.termsQuery("datasets.template", criteria.getDatasetTemplates().stream().map(UUID::toString).collect(Collectors.toList())));
             }
 
             if (criteria.getStatus() != null) {
-            boolQuery = boolQuery.should(QueryBuilders.termQuery("status.keyword", criteria.getStatus().toString()));
+                boolQuery = boolQuery.should(QueryBuilders.termQuery("datasets.status", criteria.getStatus().toString()));
             }
 
             if (criteria.getDmps() != null && criteria.getDmps().size() > 0) {
-            boolQuery = boolQuery.should(QueryBuilders.termsQuery("dmp.keyword", criteria.getDmps().stream().map(UUID::toString).collect(Collectors.toList())));
+                boolQuery = boolQuery.should(QueryBuilders.termsQuery("datasets.dmp", criteria.getDmps().stream().map(UUID::toString).collect(Collectors.toList())));
             }
 
             if (criteria.getGroupIds() != null && criteria.getGroupIds().size() > 0) {
-            boolQuery = boolQuery.should(QueryBuilders.termsQuery("group.keyword", criteria.getGroupIds().stream().map(UUID::toString).collect(Collectors.toList())));
+                boolQuery = boolQuery.should(QueryBuilders.termsQuery("datasets.group", criteria.getGroupIds().stream().map(UUID::toString).collect(Collectors.toList())));
             }
 
             if (criteria.getGrants() != null && criteria.getGrants().size() > 0) {
-            boolQuery = boolQuery.should(QueryBuilders.termsQuery("grant.keyword", criteria.getGrants().stream().map(UUID::toString).collect(Collectors.toList())));
+                boolQuery = boolQuery.should(QueryBuilders.termsQuery("datasets.grant", criteria.getGrants().stream().map(UUID::toString).collect(Collectors.toList())));
             }
 
             if (criteria.getGrantStatus() != null) {
-            boolQuery = boolQuery.should(QueryBuilders.termQuery("grantStatus.keyword", criteria.getGrantStatus().toString()));
+                boolQuery = boolQuery.should(QueryBuilders.termQuery("datasets.grantStatus", criteria.getGrantStatus().toString()));
             }
 
             if (criteria.getCollaborators() != null && criteria.getCollaborators().size() > 0) {
-            boolQuery = boolQuery.should(QueryBuilders.termsQuery("collaborators.id.keyword", criteria.getCollaborators().stream().map(UUID::toString).collect(Collectors.toList())));
+                boolQuery = boolQuery.should(QueryBuilders.termsQuery("datasets.collaborators.id", criteria.getCollaborators().stream().map(UUID::toString).collect(Collectors.toList())));
             }
 
             if (!criteria.isPublic()) {
                 if (criteria.getAllowAllVersions() != null && !criteria.getAllowAllVersions()) {
-                boolQuery = boolQuery.should(QueryBuilders.termQuery("lastVersion.keyword", "true"));
+                    boolQuery = boolQuery.should(QueryBuilders.termQuery("datasets.lastVersion", "true"));
                 }
             }
 
             if (criteria.getOrganiztions() != null && criteria.getOrganiztions().size() > 0) {
-            boolQuery = boolQuery.should(QueryBuilders.termsQuery("organizations.id.keyword", criteria.getOrganiztions()));
+                boolQuery = boolQuery.should(QueryBuilders.termsQuery("datasets.organizations.id", criteria.getOrganiztions()));
             }
 
             if (criteria.getTags() != null && criteria.getTags().size() > 0) {
-            boolQuery = boolQuery.should(QueryBuilders.termsQuery("tags.name.keyword", criteria.getTags().stream().map(Tag::getName).collect(Collectors.toList())));
+                boolQuery = boolQuery.should(QueryBuilders.termsQuery("datasets.tags.name", criteria.getTags().stream().map(Tag::getName).collect(Collectors.toList())));
             }
 
             if (boolQuery.should().isEmpty() && boolQuery.mustNot().isEmpty()) {
@@ -125,25 +177,36 @@ public class DatasetRepository extends ElasticRepository<Dataset, DatasetCriteri
             } else {
                 boolQuery.minimumShouldMatch(boolQuery.should().size());
             }
-        searchSourceBuilder.query(boolQuery);
+
+            NestedQueryBuilder nestedQueryBuilder = QueryBuilders.nestedQuery("datasets", boolQuery, ScoreMode.Avg).innerHit(new InnerHitBuilder());
+            searchSourceBuilder.query(nestedQueryBuilder);
             searchRequest.source(searchSourceBuilder);
             SearchResponse response = this.getClient().search(searchRequest, RequestOptions.DEFAULT);
-        return Arrays.stream(response.getHits().getHits()).map(x -> this.transformFromString(x.getSourceAsString(), Dataset.class)).collect(Collectors.toList());
+            return ((Stream<Dataset>)Arrays.stream(response.getHits().getHits())
+                                           .map(hit -> hit.getInnerHits().values()).flatMap(Collection::stream)
+                                           .map(SearchHits::getHits).flatMap(Arrays::stream)
+                                           .map(x -> new Dataset().fromElasticEntity(this.transformFromString(x.getSourceAsString(), Map.class)))).collect(Collectors.toList());
+        }
+        return null;
     }
 
     @Override
     public boolean exists() throws IOException {
-        GetIndexRequest request = new GetIndexRequest("datasets");
+        if (this.getClient() != null) {
+            GetIndexRequest request = new GetIndexRequest("dmps");
 //        request.indices("datasets");
             return this.getClient().indices().exists(request, RequestOptions.DEFAULT);
         }
+        return false;
+    }
 
     @Override
     public void clear() throws IOException {
-        if (exists()) {
+        //DON'T
+       /* if (exists()) {
             DeleteByQueryRequest delete = new DeleteByQueryRequest("datasets");
             delete.setQuery(QueryBuilders.matchAllQuery());
             this.getClient().deleteByQuery(delete, RequestOptions.DEFAULT);
-        }
+        }*/
     }
 }