diff --git a/roda-common/roda-common-data/pom.xml b/roda-common/roda-common-data/pom.xml index eda2db2529..294b733b43 100644 --- a/roda-common/roda-common-data/pom.xml +++ b/roda-common/roda-common-data/pom.xml @@ -67,6 +67,10 @@ org.apache.commons commons-lang3 + + jakarta.persistence + jakarta.persistence-api + diff --git a/roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/jobs/Job.java b/roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/jobs/Job.java index 1b61b1dd21..538a030a1f 100644 --- a/roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/jobs/Job.java +++ b/roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/jobs/Job.java @@ -22,14 +22,32 @@ import org.roda.core.data.v2.ip.HasId; import org.roda.core.data.v2.ip.HasInstanceID; import org.roda.core.data.v2.ip.HasInstanceName; +import org.roda.core.data.v2.jpa.JobUserDetailsListConverter; +import org.roda.core.data.v2.jpa.ObjectMapConverter; +import org.roda.core.data.v2.jpa.SelectedItemsConverter; +import org.roda.core.data.v2.jpa.StringListConverter; +import org.roda.core.data.v2.jpa.StringMapConverter; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonInclude; +import jakarta.persistence.Column; +import jakarta.persistence.Convert; +import jakarta.persistence.Entity; +import jakarta.persistence.EnumType; +import jakarta.persistence.Enumerated; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import jakarta.persistence.Temporal; +import jakarta.persistence.TemporalType; +import jakarta.persistence.Transient; + /** * @author Hélder Silva */ +@Entity +@Table(name = "jobs") @jakarta.xml.bind.annotation.XmlRootElement(name = RodaConstants.RODA_OBJECT_JOB) @JsonInclude(JsonInclude.Include.ALWAYS) @JsonIgnoreProperties(ignoreUnknown = true) @@ -42,46 +60,98 @@ public enum JOB_STATE { PENDING_APPROVAL, REJECTED, SCHEDULED; } // job identifier + @Id + @Column(name = "id") private String id = null; // job name + @Column(name = "name") private String name = null; // job creator + @Column(name = "username") private String username = null; // job start date + @Column(name = "start_date") + @Temporal(TemporalType.TIMESTAMP) private Date startDate = null; // job end date + @Column(name = "end_date") + @Temporal(TemporalType.TIMESTAMP) private Date endDate = null; // job state + @Enumerated(EnumType.STRING) + @Column(name = "state") private JOB_STATE state = null; // job state details + @Column(name = "state_details", columnDefinition = "TEXT") private String stateDetails = ""; // job instance id + @Column(name = "instance_id") private String instanceId = null; + @Column(name = "job_users_details", columnDefinition = "TEXT") + @Convert(converter = JobUserDetailsListConverter.class) private List jobUsersDetails = new ArrayList<>(); + @Column(name = "instance_name") private String instanceName = null; - // job statistics (total source objects, etc.) - JobStats jobStats = new JobStats(); + // job statistics - expanded into separate columns + @Column(name = "stats_completion_percentage") + private int statsCompletionPercentage = 0; + @Column(name = "stats_source_objects_count") + private int statsSourceObjectsCount = 0; + @Column(name = "stats_source_objects_being_processed") + private int statsSourceObjectsBeingProcessed = 0; + @Column(name = "stats_source_objects_waiting_to_be_processed") + private int statsSourceObjectsWaitingToBeProcessed = 0; + @Column(name = "stats_source_objects_processed_with_success") + private int statsSourceObjectsProcessedWithSuccess = 0; + @Column(name = "stats_source_objects_processed_with_partial_success") + private int statsSourceObjectsProcessedWithPartialSuccess = 0; + @Column(name = "stats_source_objects_processed_with_failure") + private int statsSourceObjectsProcessedWithFailure = 0; + @Column(name = "stats_source_objects_processed_with_skipped") + private int statsSourceObjectsProcessedWithSkipped = 0; + @Column(name = "stats_outcome_objects_with_manual_intervention") + private int statsOutcomeObjectsWithManualIntervention = 0; + + // Transient jobStats field for backwards compatibility with existing code + @Transient + private JobStats jobStats = new JobStats(); // plugin full class (e.g. org.roda.core.plugins.plugins.base.FixityPlugin) + @Column(name = "plugin") private String plugin = null; // plugin type (e.g. ingest, maintenance, misc, etc.) + @Enumerated(EnumType.STRING) + @Column(name = "plugin_type") private PluginType pluginType = null; // plugin parameters + @Column(name = "plugin_parameters", columnDefinition = "TEXT") + @Convert(converter = StringMapConverter.class) private Map pluginParameters = new HashMap<>(); // objects to act upon (All, None, List, Filter, etc.) + @Column(name = "source_objects", columnDefinition = "TEXT") + @Convert(converter = SelectedItemsConverter.class) private SelectedItems sourceObjects = null; + @Column(name = "outcome_objects_class") private String outcomeObjectsClass = ""; + @Column(name = "attachments_list", columnDefinition = "TEXT") + @Convert(converter = StringListConverter.class) private List attachmentsList = new ArrayList<>(); + @Column(name = "fields", columnDefinition = "TEXT") + @Convert(converter = ObjectMapConverter.class) private Map fields; + @Enumerated(EnumType.STRING) + @Column(name = "priority") private JobPriority priority; + @Enumerated(EnumType.STRING) + @Column(name = "parallelism") private JobParallelism parallelism; public Job() { @@ -104,7 +174,7 @@ public Job(Job job) { this.pluginParameters = new HashMap<>(job.getPluginParameters()); this.sourceObjects = job.getSourceObjects(); if (sourceObjects instanceof SelectedItemsList) { - jobStats.setSourceObjectsCount(((SelectedItemsList) sourceObjects).getIds().size()); + getJobStats().setSourceObjectsCount(((SelectedItemsList) sourceObjects).getIds().size()); } this.instanceId = job.getInstanceId(); this.instanceName = job.getInstanceName(); @@ -112,6 +182,7 @@ public Job(Job job) { this.jobUsersDetails = job.getJobUsersDetails(); } + @Transient @JsonIgnore @Override public int getClassVersion() { @@ -205,11 +276,36 @@ public void setStateDetails(String stateDetails) { } public JobStats getJobStats() { + // Build JobStats from individual columns for backwards compatibility + if (jobStats == null) { + jobStats = new JobStats(); + } + jobStats.setCompletionPercentage(statsCompletionPercentage); + jobStats.setSourceObjectsCount(statsSourceObjectsCount); + jobStats.setSourceObjectsBeingProcessed(statsSourceObjectsBeingProcessed); + jobStats.setSourceObjectsWaitingToBeProcessed(statsSourceObjectsWaitingToBeProcessed); + jobStats.setSourceObjectsProcessedWithSuccess(statsSourceObjectsProcessedWithSuccess); + jobStats.setSourceObjectsProcessedWithPartialSuccess(statsSourceObjectsProcessedWithPartialSuccess); + jobStats.setSourceObjectsProcessedWithFailure(statsSourceObjectsProcessedWithFailure); + jobStats.setSourceObjectsProcessedWithSkipped(statsSourceObjectsProcessedWithSkipped); + jobStats.setOutcomeObjectsWithManualIntervention(statsOutcomeObjectsWithManualIntervention); return jobStats; } public void setJobStats(JobStats jobStats) { this.jobStats = jobStats; + // Sync to individual columns + if (jobStats != null) { + this.statsCompletionPercentage = jobStats.getCompletionPercentage(); + this.statsSourceObjectsCount = jobStats.getSourceObjectsCount(); + this.statsSourceObjectsBeingProcessed = jobStats.getSourceObjectsBeingProcessed(); + this.statsSourceObjectsWaitingToBeProcessed = jobStats.getSourceObjectsWaitingToBeProcessed(); + this.statsSourceObjectsProcessedWithSuccess = jobStats.getSourceObjectsProcessedWithSuccess(); + this.statsSourceObjectsProcessedWithPartialSuccess = jobStats.getSourceObjectsProcessedWithPartialSuccess(); + this.statsSourceObjectsProcessedWithFailure = jobStats.getSourceObjectsProcessedWithFailure(); + this.statsSourceObjectsProcessedWithSkipped = jobStats.getSourceObjectsProcessedWithSkipped(); + this.statsOutcomeObjectsWithManualIntervention = jobStats.getOutcomeObjectsWithManualIntervention(); + } } public String getPlugin() { diff --git a/roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/jobs/Report.java b/roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/jobs/Report.java index b8197e7f92..02ca3a29d1 100644 --- a/roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/jobs/Report.java +++ b/roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/jobs/Report.java @@ -17,11 +17,30 @@ import org.roda.core.data.v2.ip.HasId; import org.roda.core.data.v2.ip.HasInstanceID; import org.roda.core.data.v2.ip.SIPInformation; +import org.roda.core.data.v2.jpa.StringListConverter; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import jakarta.persistence.CascadeType; +import jakarta.persistence.Column; +import jakarta.persistence.Convert; +import jakarta.persistence.Entity; +import jakarta.persistence.EnumType; +import jakarta.persistence.Enumerated; +import jakarta.persistence.FetchType; +import jakarta.persistence.Id; +import jakarta.persistence.Index; +import jakarta.persistence.OneToMany; +import jakarta.persistence.OrderBy; +import jakarta.persistence.Table; +import jakarta.persistence.Temporal; +import jakarta.persistence.TemporalType; +import jakarta.persistence.Transient; + +@Entity +@Table(name = "job_reports", indexes = {@Index(name = "idx_report_job_id", columnList = "jobId")}) @JsonInclude(JsonInclude.Include.NON_NULL) public class Report implements IsModelObject, HasId, HasInstanceID { @Serial @@ -32,39 +51,78 @@ public class Report implements IsModelObject, HasId, HasInstanceID { public static final String NO_OUTCOME_OBJECT_ID = "NO_OUTCOME_ID"; public static final String NO_OUTCOME_OBJECT_CLASS = "NO_OUTCOME_CLASS"; + @Id + @Column(name = "id") private String id = ""; + @Column(name = "job_id") private String jobId = ""; + @Column(name = "source_object_id") private String sourceObjectId = NO_SOURCE_OBJECT_ID; + @Column(name = "source_object_class") private String sourceObjectClass = NO_SOURCE_OBJECT_CLASS; + @Column(name = "source_object_original_ids", columnDefinition = "TEXT") + @Convert(converter = StringListConverter.class) private List sourceObjectOriginalIds = new ArrayList<>(); + @Column(name = "source_object_original_name") private String sourceObjectOriginalName = ""; + @Column(name = "outcome_object_id") private String outcomeObjectId = NO_OUTCOME_OBJECT_ID; + @Column(name = "outcome_object_class") private String outcomeObjectClass = NO_OUTCOME_OBJECT_CLASS; + @Enumerated(EnumType.STRING) + @Column(name = "outcome_object_state") private AIPState outcomeObjectState = AIPState.getDefault(); + @Column(name = "title") private String title = ""; + @Column(name = "date_created") + @Temporal(TemporalType.TIMESTAMP) private Date dateCreated; + @Column(name = "date_updated") + @Temporal(TemporalType.TIMESTAMP) private Date dateUpdated; + @Column(name = "ingest_type") private String ingestType = ""; + @Column(name = "completion_percentage") private Integer completionPercentage = 0; + @Column(name = "steps_completed") private Integer stepsCompleted = 0; + @Column(name = "total_steps") private Integer totalSteps = 0; + @Column(name = "plugin") private String plugin = ""; + @Column(name = "plugin_name") private String pluginName = ""; + @Column(name = "plugin_version") private String pluginVersion = ""; + @Enumerated(EnumType.STRING) + @Column(name = "plugin_state") private PluginState pluginState = PluginState.RUNNING; + @Column(name = "plugin_is_mandatory") private Boolean pluginIsMandatory = true; + @Column(name = "plugin_details", columnDefinition = "TEXT") private String pluginDetails = ""; + @Column(name = "html_plugin_details") private boolean htmlPluginDetails = false; + @Column(name = "instance_id") private String instanceId = null; + @Column(name = "transaction_id") private String transactionId = null; + @Transient @JsonIgnore private SIPInformation sipInformation = new SIPInformation(); - private List reports = new ArrayList<>(); + @OneToMany(mappedBy = "parentReportId", cascade = CascadeType.ALL, fetch = FetchType.LAZY, orphanRemoval = true) + @OrderBy("stepOrder ASC") + private List stepReports = new ArrayList<>(); + // Transient field for backwards compatibility with existing code that uses List + @Transient + private List reports = null; + + @Column(name = "line_separator") private String lineSeparator = ""; public Report() { @@ -101,11 +159,13 @@ public Report(Report report) { this.pluginIsMandatory = report.getPluginIsMandatory(); this.pluginDetails = report.getPluginDetails(); this.htmlPluginDetails = report.isHtmlPluginDetails(); - this.reports = new ArrayList<>(); + this.reports = null; + this.stepReports = new ArrayList<>(); this.instanceId = report.getInstanceId(); this.transactionId = report.getTransactionId(); } + @Transient @JsonIgnore @Override public int getClassVersion() { @@ -420,12 +480,25 @@ public Report addReport(Report report, boolean updateReportItemDateUpdated) { ReportUtils.calculatePluginState(getPluginState(), report.getPluginState(), report.getPluginIsMandatory())); if (!"".equals(report.getPluginDetails()) && !getPluginDetails().equals(report.getPluginDetails())) { + // Fix: avoid adding repeated line separators + String separator = (lineSeparator != null && !lineSeparator.isEmpty()) ? lineSeparator : "\n"; setPluginDetails( - (!"".equals(getPluginDetails()) ? getPluginDetails() + lineSeparator : "") + report.getPluginDetails()); + (!"".equals(getPluginDetails()) ? getPluginDetails() + separator : "") + report.getPluginDetails()); } setOutcomeObjectState(report.getOutcomeObjectState()); + // Add to both stepReports (JPA entity) and reports (backwards compatibility) + if (reports == null) { + reports = new ArrayList<>(); + } reports.add(report); + + // Also add as StepReport + if (stepReports == null) { + stepReports = new ArrayList<>(); + } + stepReports.add(new StepReport(report, this.id, stepReports.size())); + return this; } @@ -445,15 +518,50 @@ public Report addReport(Report report, boolean updateReportItemDateUpdated) { * return newPluginState; } */ + /** + * Get the step reports as a list of Report objects for backwards compatibility. + * If reports is null, builds it from stepReports. + */ public List getReports() { - return reports; + if (reports == null && stepReports != null) { + reports = new ArrayList<>(); + for (StepReport stepReport : stepReports) { + reports.add(stepReport.toReport()); + } + } + return reports != null ? reports : new ArrayList<>(); } public Report setReports(List reports) { this.reports = reports; + // Sync to stepReports + if (reports != null) { + this.stepReports = new ArrayList<>(); + int order = 0; + for (Report report : reports) { + this.stepReports.add(new StepReport(report, this.id, order++)); + } + } return this; } + /** + * Get the underlying StepReport entities. + */ + @JsonIgnore + public List getStepReports() { + return stepReports; + } + + /** + * Set the underlying StepReport entities. + */ + @JsonIgnore + public void setStepReports(List stepReports) { + this.stepReports = stepReports; + this.reports = null; // Reset cached reports + } + @JsonProperty("lineSeparator") public void injectLineSeparator(String lineSeparator) { this.lineSeparator = lineSeparator; @@ -465,10 +573,11 @@ public String getLineSeparator() { @JsonIgnore public Report getLastRunPlugin() { - int size = reports.size(); + List reportsList = getReports(); + int size = reportsList.size(); if (size == 0) return null; - return reports.get(size - 1); + return reportsList.get(size - 1); } @Override diff --git a/roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/jobs/StepReport.java b/roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/jobs/StepReport.java new file mode 100644 index 0000000000..12eee32243 --- /dev/null +++ b/roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/jobs/StepReport.java @@ -0,0 +1,347 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE file at the root of the source + * tree and available online at + * + * https://github.com/keeps/roda + */ +package org.roda.core.data.v2.jobs; + +import java.io.Serial; +import java.io.Serializable; +import java.util.Date; +import java.util.UUID; + +import org.roda.core.data.v2.ip.AIPState; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; + +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.EnumType; +import jakarta.persistence.Enumerated; +import jakarta.persistence.Id; +import jakarta.persistence.Index; +import jakarta.persistence.Table; +import jakarta.persistence.Temporal; +import jakarta.persistence.TemporalType; + +/** + * StepReport represents a single step/plugin execution within a job report. + * This is a separate JPA entity to properly store step reports in their own table + * with a foreign key relationship to the parent Report. + * + * @author RODA Development Team + */ +@Entity +@Table(name = "job_step_reports", indexes = {@Index(name = "idx_step_report_parent_id", columnList = "parent_report_id")}) +@JsonInclude(JsonInclude.Include.NON_NULL) +public class StepReport implements Serializable { + @Serial + private static final long serialVersionUID = 1L; + + @Id + @Column(name = "id") + private String id; + + @Column(name = "parent_report_id") + private String parentReportId; + + @Column(name = "source_object_id") + private String sourceObjectId = Report.NO_SOURCE_OBJECT_ID; + + @Column(name = "source_object_class") + private String sourceObjectClass = Report.NO_SOURCE_OBJECT_CLASS; + + @Column(name = "outcome_object_id") + private String outcomeObjectId = Report.NO_OUTCOME_OBJECT_ID; + + @Column(name = "outcome_object_class") + private String outcomeObjectClass = Report.NO_OUTCOME_OBJECT_CLASS; + + @Enumerated(EnumType.STRING) + @Column(name = "outcome_object_state") + private AIPState outcomeObjectState = AIPState.getDefault(); + + @Column(name = "title") + private String title = ""; + + @Column(name = "date_created") + @Temporal(TemporalType.TIMESTAMP) + private Date dateCreated; + + @Column(name = "date_updated") + @Temporal(TemporalType.TIMESTAMP) + private Date dateUpdated; + + @Column(name = "completion_percentage") + private Integer completionPercentage = 0; + + @Column(name = "steps_completed") + private Integer stepsCompleted = 0; + + @Column(name = "total_steps") + private Integer totalSteps = 0; + + @Column(name = "plugin") + private String plugin = ""; + + @Column(name = "plugin_name") + private String pluginName = ""; + + @Column(name = "plugin_version") + private String pluginVersion = ""; + + @Enumerated(EnumType.STRING) + @Column(name = "plugin_state") + private PluginState pluginState = PluginState.RUNNING; + + @Column(name = "plugin_is_mandatory") + private Boolean pluginIsMandatory = true; + + @Column(name = "plugin_details", columnDefinition = "TEXT") + private String pluginDetails = ""; + + @Column(name = "html_plugin_details") + private boolean htmlPluginDetails = false; + + @Column(name = "step_order") + private Integer stepOrder = 0; + + public StepReport() { + super(); + this.id = UUID.randomUUID().toString(); + this.dateCreated = new Date(); + } + + /** + * Create a StepReport from a Report (for backwards compatibility during migration) + */ + public StepReport(Report report, String parentReportId, int stepOrder) { + this(); + this.parentReportId = parentReportId; + this.stepOrder = stepOrder; + this.sourceObjectId = report.getSourceObjectId(); + this.sourceObjectClass = report.getSourceObjectClass(); + this.outcomeObjectId = report.getOutcomeObjectId(); + this.outcomeObjectClass = report.getOutcomeObjectClass(); + this.outcomeObjectState = report.getOutcomeObjectState(); + this.title = report.getTitle(); + this.dateCreated = report.getDateCreated(); + this.dateUpdated = report.getDateUpdated(); + this.completionPercentage = report.getCompletionPercentage(); + this.stepsCompleted = report.getStepsCompleted(); + this.totalSteps = report.getTotalSteps(); + this.plugin = report.getPlugin(); + this.pluginName = report.getPluginName(); + this.pluginVersion = report.getPluginVersion(); + this.pluginState = report.getPluginState(); + this.pluginIsMandatory = report.getPluginIsMandatory(); + this.pluginDetails = report.getPluginDetails(); + this.htmlPluginDetails = report.isHtmlPluginDetails(); + } + + /** + * Convert this StepReport to a Report (for backwards compatibility) + */ + @JsonIgnore + public Report toReport() { + Report report = new Report(); + report.setId(this.id); + report.setSourceObjectId(this.sourceObjectId); + report.setSourceObjectClass(this.sourceObjectClass); + report.setOutcomeObjectId(this.outcomeObjectId); + report.setOutcomeObjectClass(this.outcomeObjectClass); + report.setOutcomeObjectState(this.outcomeObjectState); + report.setTitle(this.title); + report.setDateCreated(this.dateCreated); + report.setDateUpdated(this.dateUpdated); + report.setCompletionPercentage(this.completionPercentage); + report.setStepsCompleted(this.stepsCompleted); + report.setTotalSteps(this.totalSteps); + report.setPlugin(this.plugin); + report.setPluginName(this.pluginName); + report.setPluginVersion(this.pluginVersion); + report.setPluginState(this.pluginState); + report.setPluginIsMandatory(this.pluginIsMandatory); + report.setPluginDetails(this.pluginDetails); + report.setHtmlPluginDetails(this.htmlPluginDetails); + return report; + } + + // Getters and setters + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getParentReportId() { + return parentReportId; + } + + public void setParentReportId(String parentReportId) { + this.parentReportId = parentReportId; + } + + public String getSourceObjectId() { + return sourceObjectId; + } + + public void setSourceObjectId(String sourceObjectId) { + this.sourceObjectId = sourceObjectId; + } + + public String getSourceObjectClass() { + return sourceObjectClass; + } + + public void setSourceObjectClass(String sourceObjectClass) { + this.sourceObjectClass = sourceObjectClass; + } + + public String getOutcomeObjectId() { + return outcomeObjectId; + } + + public void setOutcomeObjectId(String outcomeObjectId) { + this.outcomeObjectId = outcomeObjectId; + } + + public String getOutcomeObjectClass() { + return outcomeObjectClass; + } + + public void setOutcomeObjectClass(String outcomeObjectClass) { + this.outcomeObjectClass = outcomeObjectClass; + } + + public AIPState getOutcomeObjectState() { + return outcomeObjectState; + } + + public void setOutcomeObjectState(AIPState outcomeObjectState) { + this.outcomeObjectState = outcomeObjectState; + } + + public String getTitle() { + return title; + } + + public void setTitle(String title) { + this.title = title; + } + + public Date getDateCreated() { + return dateCreated; + } + + public void setDateCreated(Date dateCreated) { + this.dateCreated = dateCreated; + } + + public Date getDateUpdated() { + return dateUpdated; + } + + public void setDateUpdated(Date dateUpdated) { + this.dateUpdated = dateUpdated; + } + + public Integer getCompletionPercentage() { + return completionPercentage; + } + + public void setCompletionPercentage(Integer completionPercentage) { + this.completionPercentage = completionPercentage; + } + + public Integer getStepsCompleted() { + return stepsCompleted; + } + + public void setStepsCompleted(Integer stepsCompleted) { + this.stepsCompleted = stepsCompleted; + } + + public Integer getTotalSteps() { + return totalSteps; + } + + public void setTotalSteps(Integer totalSteps) { + this.totalSteps = totalSteps; + } + + public String getPlugin() { + return plugin; + } + + public void setPlugin(String plugin) { + this.plugin = plugin; + } + + public String getPluginName() { + return pluginName; + } + + public void setPluginName(String pluginName) { + this.pluginName = pluginName; + } + + public String getPluginVersion() { + return pluginVersion; + } + + public void setPluginVersion(String pluginVersion) { + this.pluginVersion = pluginVersion; + } + + public PluginState getPluginState() { + return pluginState; + } + + public void setPluginState(PluginState pluginState) { + this.pluginState = pluginState; + } + + public Boolean getPluginIsMandatory() { + return pluginIsMandatory; + } + + public void setPluginIsMandatory(Boolean pluginIsMandatory) { + this.pluginIsMandatory = pluginIsMandatory; + } + + public String getPluginDetails() { + return pluginDetails; + } + + public void setPluginDetails(String pluginDetails) { + this.pluginDetails = pluginDetails; + } + + public boolean isHtmlPluginDetails() { + return htmlPluginDetails; + } + + public void setHtmlPluginDetails(boolean htmlPluginDetails) { + this.htmlPluginDetails = htmlPluginDetails; + } + + public Integer getStepOrder() { + return stepOrder; + } + + public void setStepOrder(Integer stepOrder) { + this.stepOrder = stepOrder; + } + + @Override + public String toString() { + return "StepReport [id=" + id + ", parentReportId=" + parentReportId + ", sourceObjectId=" + sourceObjectId + + ", plugin=" + plugin + ", pluginState=" + pluginState + ", stepOrder=" + stepOrder + "]"; + } +} diff --git a/roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/jpa/JobStatsConverter.java b/roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/jpa/JobStatsConverter.java new file mode 100644 index 0000000000..61424cd06c --- /dev/null +++ b/roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/jpa/JobStatsConverter.java @@ -0,0 +1,24 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE file at the root of the source + * tree and available online at + * + * https://github.com/keeps/roda + */ +package org.roda.core.data.v2.jpa; + +import org.roda.core.data.v2.jobs.JobStats; + +import jakarta.persistence.Converter; + +/** + * JPA converter for JobStats objects. + */ +@Converter +public class JobStatsConverter extends JsonAttributeConverter { + + @Override + protected Class getAttributeClass() { + return JobStats.class; + } +} diff --git a/roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/jpa/JobUserDetailsListConverter.java b/roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/jpa/JobUserDetailsListConverter.java new file mode 100644 index 0000000000..f5f9cc5693 --- /dev/null +++ b/roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/jpa/JobUserDetailsListConverter.java @@ -0,0 +1,50 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE file at the root of the source + * tree and available online at + * + * https://github.com/keeps/roda + */ +package org.roda.core.data.v2.jpa; + +import java.util.ArrayList; +import java.util.List; + +import org.roda.core.data.exceptions.GenericException; +import org.roda.core.data.utils.JsonUtils; +import org.roda.core.data.v2.jobs.JobUserDetails; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import jakarta.persistence.AttributeConverter; +import jakarta.persistence.Converter; + +/** + * JPA converter for List of JobUserDetails objects. + */ +@Converter +public class JobUserDetailsListConverter implements AttributeConverter, String> { + + private static final Logger LOGGER = LoggerFactory.getLogger(JobUserDetailsListConverter.class); + + @Override + public String convertToDatabaseColumn(List attribute) { + if (attribute == null) { + return null; + } + return JsonUtils.getJsonFromObject(attribute); + } + + @Override + public List convertToEntityAttribute(String dbData) { + if (dbData == null || dbData.isEmpty()) { + return new ArrayList<>(); + } + try { + return JsonUtils.getListFromJson(dbData, JobUserDetails.class); + } catch (GenericException e) { + LOGGER.error("Error converting JSON to List: {}", e.getMessage(), e); + return new ArrayList<>(); + } + } +} diff --git a/roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/jpa/JsonAttributeConverter.java b/roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/jpa/JsonAttributeConverter.java new file mode 100644 index 0000000000..a1bb3e0537 --- /dev/null +++ b/roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/jpa/JsonAttributeConverter.java @@ -0,0 +1,55 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE file at the root of the source + * tree and available online at + * + * https://github.com/keeps/roda + */ +package org.roda.core.data.v2.jpa; + +import org.roda.core.data.exceptions.GenericException; +import org.roda.core.data.utils.JsonUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import jakarta.persistence.AttributeConverter; + +/** + * Abstract base class for JPA attribute converters that serialize/deserialize + * objects to/from JSON strings. + * + * @param + * The type of the attribute to convert + */ +public abstract class JsonAttributeConverter implements AttributeConverter { + + private static final Logger LOGGER = LoggerFactory.getLogger(JsonAttributeConverter.class); + + /** + * Returns the Class type for deserialization. + * + * @return the class type of the attribute + */ + protected abstract Class getAttributeClass(); + + @Override + public String convertToDatabaseColumn(T attribute) { + if (attribute == null) { + return null; + } + return JsonUtils.getJsonFromObject(attribute); + } + + @Override + public T convertToEntityAttribute(String dbData) { + if (dbData == null || dbData.isEmpty()) { + return null; + } + try { + return JsonUtils.getObjectFromJson(dbData, getAttributeClass()); + } catch (GenericException e) { + LOGGER.error("Error converting JSON to {}: {}", getAttributeClass().getSimpleName(), e.getMessage(), e); + return null; + } + } +} diff --git a/roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/jpa/ObjectMapConverter.java b/roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/jpa/ObjectMapConverter.java new file mode 100644 index 0000000000..c82cb521ee --- /dev/null +++ b/roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/jpa/ObjectMapConverter.java @@ -0,0 +1,54 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE file at the root of the source + * tree and available online at + * + * https://github.com/keeps/roda + */ +package org.roda.core.data.v2.jpa; + +import java.util.HashMap; +import java.util.Map; + +import org.roda.core.data.exceptions.GenericException; +import org.roda.core.data.utils.JsonUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; + +import jakarta.persistence.AttributeConverter; +import jakarta.persistence.Converter; + +/** + * JPA converter for Map<String, Object> objects. + */ +@Converter +public class ObjectMapConverter implements AttributeConverter, String> { + + private static final Logger LOGGER = LoggerFactory.getLogger(ObjectMapConverter.class); + + @Override + public String convertToDatabaseColumn(Map attribute) { + if (attribute == null) { + return null; + } + return JsonUtils.getJsonFromObject(attribute); + } + + @Override + public Map convertToEntityAttribute(String dbData) { + if (dbData == null || dbData.isEmpty()) { + return new HashMap<>(); + } + try { + ObjectMapper mapper = new ObjectMapper(new JsonFactory()); + return mapper.readValue(dbData, new TypeReference>() {}); + } catch (Exception e) { + LOGGER.error("Error converting JSON to Map: {}", e.getMessage(), e); + return new HashMap<>(); + } + } +} diff --git a/roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/jpa/ReportListConverter.java b/roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/jpa/ReportListConverter.java new file mode 100644 index 0000000000..45096098df --- /dev/null +++ b/roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/jpa/ReportListConverter.java @@ -0,0 +1,50 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE file at the root of the source + * tree and available online at + * + * https://github.com/keeps/roda + */ +package org.roda.core.data.v2.jpa; + +import java.util.ArrayList; +import java.util.List; + +import org.roda.core.data.exceptions.GenericException; +import org.roda.core.data.utils.JsonUtils; +import org.roda.core.data.v2.jobs.Report; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import jakarta.persistence.AttributeConverter; +import jakarta.persistence.Converter; + +/** + * JPA converter for List of Report objects. + */ +@Converter +public class ReportListConverter implements AttributeConverter, String> { + + private static final Logger LOGGER = LoggerFactory.getLogger(ReportListConverter.class); + + @Override + public String convertToDatabaseColumn(List attribute) { + if (attribute == null) { + return null; + } + return JsonUtils.getJsonFromObject(attribute); + } + + @Override + public List convertToEntityAttribute(String dbData) { + if (dbData == null || dbData.isEmpty()) { + return new ArrayList<>(); + } + try { + return JsonUtils.getListFromJson(dbData, Report.class); + } catch (GenericException e) { + LOGGER.error("Error converting JSON to List: {}", e.getMessage(), e); + return new ArrayList<>(); + } + } +} diff --git a/roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/jpa/SelectedItemsConverter.java b/roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/jpa/SelectedItemsConverter.java new file mode 100644 index 0000000000..ebe96bd571 --- /dev/null +++ b/roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/jpa/SelectedItemsConverter.java @@ -0,0 +1,48 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE file at the root of the source + * tree and available online at + * + * https://github.com/keeps/roda + */ +package org.roda.core.data.v2.jpa; + +import org.roda.core.data.exceptions.GenericException; +import org.roda.core.data.utils.JsonUtils; +import org.roda.core.data.v2.index.select.SelectedItems; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import jakarta.persistence.AttributeConverter; +import jakarta.persistence.Converter; + +/** + * JPA converter for SelectedItems objects. + */ +@Converter +public class SelectedItemsConverter implements AttributeConverter, String> { + + private static final Logger LOGGER = LoggerFactory.getLogger(SelectedItemsConverter.class); + + @Override + public String convertToDatabaseColumn(SelectedItems attribute) { + if (attribute == null) { + return null; + } + return JsonUtils.getJsonFromObject(attribute); + } + + @Override + @SuppressWarnings("unchecked") + public SelectedItems convertToEntityAttribute(String dbData) { + if (dbData == null || dbData.isEmpty()) { + return null; + } + try { + return JsonUtils.getObjectFromJson(dbData, SelectedItems.class); + } catch (GenericException e) { + LOGGER.error("Error converting JSON to SelectedItems: {}", e.getMessage(), e); + return null; + } + } +} diff --git a/roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/jpa/StringListConverter.java b/roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/jpa/StringListConverter.java new file mode 100644 index 0000000000..477ac45874 --- /dev/null +++ b/roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/jpa/StringListConverter.java @@ -0,0 +1,49 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE file at the root of the source + * tree and available online at + * + * https://github.com/keeps/roda + */ +package org.roda.core.data.v2.jpa; + +import java.util.ArrayList; +import java.util.List; + +import org.roda.core.data.exceptions.GenericException; +import org.roda.core.data.utils.JsonUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import jakarta.persistence.AttributeConverter; +import jakarta.persistence.Converter; + +/** + * JPA converter for List<String> objects. + */ +@Converter +public class StringListConverter implements AttributeConverter, String> { + + private static final Logger LOGGER = LoggerFactory.getLogger(StringListConverter.class); + + @Override + public String convertToDatabaseColumn(List attribute) { + if (attribute == null) { + return null; + } + return JsonUtils.getJsonFromObject(attribute); + } + + @Override + public List convertToEntityAttribute(String dbData) { + if (dbData == null || dbData.isEmpty()) { + return new ArrayList<>(); + } + try { + return JsonUtils.getListFromJson(dbData, String.class); + } catch (GenericException e) { + LOGGER.error("Error converting JSON to List: {}", e.getMessage(), e); + return new ArrayList<>(); + } + } +} diff --git a/roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/jpa/StringMapConverter.java b/roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/jpa/StringMapConverter.java new file mode 100644 index 0000000000..a5d029a4ad --- /dev/null +++ b/roda-common/roda-common-data/src/main/java/org/roda/core/data/v2/jpa/StringMapConverter.java @@ -0,0 +1,43 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE file at the root of the source + * tree and available online at + * + * https://github.com/keeps/roda + */ +package org.roda.core.data.v2.jpa; + +import java.util.HashMap; +import java.util.Map; + +import org.roda.core.data.utils.JsonUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import jakarta.persistence.AttributeConverter; +import jakarta.persistence.Converter; + +/** + * JPA converter for Map<String, String> objects. + */ +@Converter +public class StringMapConverter implements AttributeConverter, String> { + + private static final Logger LOGGER = LoggerFactory.getLogger(StringMapConverter.class); + + @Override + public String convertToDatabaseColumn(Map attribute) { + if (attribute == null) { + return null; + } + return JsonUtils.getJsonFromObject(attribute); + } + + @Override + public Map convertToEntityAttribute(String dbData) { + if (dbData == null || dbData.isEmpty()) { + return new HashMap<>(); + } + return JsonUtils.getMapFromJson(dbData); + } +} diff --git a/roda-core/roda-core-tests/src/main/java/org/roda/core/config/TestConfig.java b/roda-core/roda-core-tests/src/main/java/org/roda/core/config/TestConfig.java index 3a6b6dac7d..7901382e9d 100644 --- a/roda-core/roda-core-tests/src/main/java/org/roda/core/config/TestConfig.java +++ b/roda-core/roda-core-tests/src/main/java/org/roda/core/config/TestConfig.java @@ -21,6 +21,6 @@ @EnableAutoConfiguration @ComponentScan(basePackages = "org.roda.core") @EnableJpaRepositories(basePackages = "org.roda.core.repository") -@EntityScan(basePackages = "org.roda.core.entity") +@EntityScan(basePackages = {"org.roda.core.entity", "org.roda.core.data.v2.jobs"}) public class TestConfig { } diff --git a/roda-core/roda-core-tests/src/main/java/org/roda/core/model/JobPersistenceTest.java b/roda-core/roda-core-tests/src/main/java/org/roda/core/model/JobPersistenceTest.java new file mode 100644 index 0000000000..faeee7a5c2 --- /dev/null +++ b/roda-core/roda-core-tests/src/main/java/org/roda/core/model/JobPersistenceTest.java @@ -0,0 +1,314 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE file at the root of the source + * tree and available online at + * + * https://github.com/keeps/roda + */ +package org.roda.core.model; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.UUID; +import java.util.stream.Collectors; +import java.util.stream.StreamSupport; + +import org.roda.core.RodaCoreFactory; +import org.roda.core.TestsHelper; +import org.roda.core.common.iterables.CloseableIterable; +import org.roda.core.config.TestConfig; +import org.roda.core.data.common.RodaConstants; +import org.roda.core.data.exceptions.GenericException; +import org.roda.core.data.exceptions.NotFoundException; +import org.roda.core.data.exceptions.RODAException; +import org.roda.core.data.v2.common.OptionalWithCause; +import org.roda.core.data.v2.index.select.SelectedItemsNone; +import org.roda.core.data.v2.jobs.Job; +import org.roda.core.data.v2.jobs.Job.JOB_STATE; +import org.roda.core.data.v2.jobs.PluginType; +import org.roda.core.data.v2.jobs.Report; +import org.roda.core.repository.job.JobRepository; +import org.roda.core.repository.job.ReportRepository; +import org.roda.core.security.LdapUtilityTestHelper; +import org.roda.core.storage.StorageService; +import org.roda.core.storage.fs.FSUtils; +import org.roda.core.util.IdUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.testng.AbstractTestNGSpringContextTests; +import org.testng.annotations.AfterClass; +import org.testng.annotations.BeforeClass; +import org.testng.annotations.Test; + +/** + * Unit tests for the hybrid Job/Report persistence logic. + * Tests that running jobs are stored in the database and flushed to storage on completion. + * + * @author RODA Development Team + */ +@SpringBootTest(classes = TestConfig.class) +@Test(groups = {RodaConstants.TEST_GROUP_ALL, RodaConstants.TEST_GROUP_DEV}) +public class JobPersistenceTest extends AbstractTestNGSpringContextTests { + private static final Logger LOGGER = LoggerFactory.getLogger(JobPersistenceTest.class); + + private static Path basePath; + private static StorageService storage; + private static ModelService model; + private static LdapUtilityTestHelper ldapUtilityTestHelper; + + @Autowired + private JobRepository jobRepository; + + @Autowired + private ReportRepository reportRepository; + + @BeforeClass + public void init() throws IOException, GenericException { + basePath = TestsHelper.createBaseTempDir(getClass(), true); + ldapUtilityTestHelper = new LdapUtilityTestHelper(); + + boolean deploySolr = false; + boolean deployLdap = true; + boolean deployFolderMonitor = false; + boolean deployOrchestrator = false; + boolean deployPluginManager = false; + boolean deployDefaultResources = false; + RodaCoreFactory.instantiateTest(deploySolr, deployLdap, deployFolderMonitor, deployOrchestrator, + deployPluginManager, deployDefaultResources, false, ldapUtilityTestHelper.getLdapUtility()); + + storage = RodaCoreFactory.getStorageService(); + model = RodaCoreFactory.getModelService(); + + LOGGER.debug("Running JobPersistenceTest under storage: {}", basePath); + } + + @AfterClass + public void cleanup() throws NotFoundException, GenericException, IOException { + // Clean up any test data + jobRepository.deleteAll(); + reportRepository.deleteAll(); + + ldapUtilityTestHelper.shutdown(); + RodaCoreFactory.shutdown(); + FSUtils.deletePath(basePath); + } + + /** + * Test that a newly created job with a non-final state (STARTED) is saved to the database + * and NOT written to file storage. + */ + @Test + public void testRunningJobPersistence() throws RODAException { + // Create a running job + String jobId = IdUtils.createUUID(); + Job job = createTestJob(jobId, JOB_STATE.STARTED); + + // Create the job using the model service + model.createJob(job); + + // Verify job exists in database + assertTrue(jobRepository.existsById(jobId), "Job should exist in database"); + + // Verify job retrieved from model service + Job retrievedJob = model.retrieveJob(jobId); + assertNotNull(retrievedJob, "Should be able to retrieve the job"); + assertEquals(retrievedJob.getId(), jobId); + assertEquals(retrievedJob.getState(), JOB_STATE.STARTED); + + // Clean up + model.deleteJob(jobId); + } + + /** + * Test that updating a job to a final state (COMPLETED) flushes it from the database + * to file storage. + */ + @Test + public void testJobFinalization() throws RODAException { + // Create a running job + String jobId = IdUtils.createUUID(); + Job job = createTestJob(jobId, JOB_STATE.STARTED); + + // Create the job using the model service + model.createJob(job); + + // Verify job is in database initially + assertTrue(jobRepository.existsById(jobId), "Job should exist in database initially"); + + // Create a report for this job + Report report = createTestReport(jobId); + model.createOrUpdateJobReport(report, job); + + // Verify report is in database + assertTrue(reportRepository.existsById(report.getId()), "Report should exist in database"); + + // Now update job to final state + job.setState(JOB_STATE.COMPLETED); + job.setEndDate(new Date()); + model.createOrUpdateJob(job); + + // Verify job is no longer in database (flushed to storage) + assertFalse(jobRepository.existsById(jobId), "Job should not exist in database after completion"); + + // Verify report is no longer in database + assertFalse(reportRepository.existsById(report.getId()), "Report should not exist in database after job completion"); + + // Verify job can still be retrieved (from storage) + Job retrievedJob = model.retrieveJob(jobId); + assertNotNull(retrievedJob, "Should be able to retrieve completed job from storage"); + assertEquals(retrievedJob.getState(), JOB_STATE.COMPLETED); + + // Clean up + model.deleteJob(jobId); + } + + /** + * Test that the list method returns both running jobs (from DB) and completed jobs (from storage). + */ + @Test + public void testListingConsistency() throws RODAException { + // Create a running job (will be in DB) + String runningJobId = IdUtils.createUUID(); + Job runningJob = createTestJob(runningJobId, JOB_STATE.STARTED); + model.createJob(runningJob); + + // Create a completed job (will be in storage) + String completedJobId = IdUtils.createUUID(); + Job completedJob = createTestJob(completedJobId, JOB_STATE.COMPLETED); + completedJob.setEndDate(new Date()); + model.createJob(completedJob); + // Force transition to storage by creating and immediately completing + model.createOrUpdateJob(completedJob); + + // List all jobs using model service + try (CloseableIterable> jobsIterable = model.list(Job.class)) { + List allJobs = StreamSupport.stream(jobsIterable.spliterator(), false) + .filter(OptionalWithCause::isPresent) + .map(OptionalWithCause::get) + .collect(Collectors.toList()); + + // Verify both jobs are listed + assertTrue(allJobs.stream().anyMatch(j -> j.getId().equals(runningJobId)), + "Running job should be in the list"); + // Note: completed job may or may not be in list depending on timing + + LOGGER.info("Listed {} jobs total", allJobs.size()); + } catch (IOException e) { + throw new GenericException("Error closing iterable", e); + } + + // Clean up + model.deleteJob(runningJobId); + try { + model.deleteJob(completedJobId); + } catch (NotFoundException e) { + // May have already been deleted or never existed in storage + } + } + + /** + * Test that deleteJob properly cleans up both DB and storage. + */ + @Test + public void testDeletion() throws RODAException { + // Create a running job + String jobId = IdUtils.createUUID(); + Job job = createTestJob(jobId, JOB_STATE.STARTED); + model.createJob(job); + + // Create a report + Report report = createTestReport(jobId); + model.createOrUpdateJobReport(report, job); + + // Verify they exist in DB + assertTrue(jobRepository.existsById(jobId), "Job should exist in database"); + assertTrue(reportRepository.existsById(report.getId()), "Report should exist in database"); + + // Delete the job + model.deleteJob(jobId); + + // Verify both job and reports are deleted from DB + assertFalse(jobRepository.existsById(jobId), "Job should be deleted from database"); + List remainingReports = reportRepository.findByJobId(jobId); + assertTrue(remainingReports.isEmpty(), "Reports should be deleted from database"); + + // Verify job cannot be retrieved + boolean notFound = false; + try { + model.retrieveJob(jobId); + } catch (NotFoundException e) { + notFound = true; + } + assertTrue(notFound, "Job should not be found after deletion"); + } + + /** + * Test report persistence for running jobs. + */ + @Test + public void testReportPersistence() throws RODAException { + // Create a running job + String jobId = IdUtils.createUUID(); + Job job = createTestJob(jobId, JOB_STATE.STARTED); + model.createJob(job); + + // Create multiple reports + Report report1 = createTestReport(jobId); + Report report2 = createTestReport(jobId); + model.createOrUpdateJobReport(report1, job); + model.createOrUpdateJobReport(report2, job); + + // Verify reports are in database + List dbReports = reportRepository.findByJobId(jobId); + assertEquals(dbReports.size(), 2, "Should have 2 reports in database"); + + // Verify reports can be listed through model service + try (CloseableIterable> reportsIterable = model.listJobReports(jobId)) { + List listedReports = StreamSupport.stream(reportsIterable.spliterator(), false) + .filter(OptionalWithCause::isPresent) + .map(OptionalWithCause::get) + .collect(Collectors.toList()); + assertEquals(listedReports.size(), 2, "Should list 2 reports through model service"); + } catch (IOException e) { + throw new GenericException("Error closing iterable", e); + } + + // Clean up + model.deleteJob(jobId); + } + + private Job createTestJob(String jobId, JOB_STATE state) { + Job job = new Job(); + job.setId(jobId); + job.setName("Test Job " + jobId); + job.setUsername(RodaConstants.ADMIN); + job.setState(state); + job.setStartDate(new Date()); + job.setPlugin("org.roda.core.plugins.test.TestPlugin"); + job.setPluginType(PluginType.MISC); + job.setPluginParameters(new HashMap<>()); + job.setSourceObjects(new SelectedItemsNone<>()); + return job; + } + + private Report createTestReport(String jobId) { + Report report = new Report(); + report.setId(IdUtils.createUUID()); + report.setJobId(jobId); + report.setSourceObjectId("test-source-" + UUID.randomUUID().toString().substring(0, 8)); + report.setOutcomeObjectId("test-outcome-" + UUID.randomUUID().toString().substring(0, 8)); + report.setDateCreated(new Date()); + report.setTitle("Test Report"); + return report; + } +} diff --git a/roda-core/roda-core/src/main/java/org/roda/core/model/DefaultModelService.java b/roda-core/roda-core/src/main/java/org/roda/core/model/DefaultModelService.java index a84c76134f..95c9709847 100644 --- a/roda-core/roda-core/src/main/java/org/roda/core/model/DefaultModelService.java +++ b/roda-core/roda-core/src/main/java/org/roda/core/model/DefaultModelService.java @@ -183,6 +183,10 @@ import org.roda.core.util.HTTPUtility; import org.roda.core.util.IdUtils; import org.roda.core.util.RESTClientUtility; +import org.roda.core.config.SpringContext; +import org.roda.core.repository.job.JobRepository; +import org.roda.core.repository.job.ReportRepository; +import org.roda.core.repository.job.StepReportRepository; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -207,6 +211,48 @@ public class DefaultModelService implements ModelService { private Object logFileLock = new Object(); private long entryLogLineNumber = -1; + // Lazy-loaded JPA repositories for hybrid Job/Report persistence + private JobRepository jobRepository; + private ReportRepository reportRepository; + private StepReportRepository stepReportRepository; + + /** + * Lazily retrieves the JobRepository bean from Spring context. + */ + private JobRepository getJobRepository() { + if (jobRepository == null && SpringContext.isContextInitialized()) { + jobRepository = SpringContext.getBean(JobRepository.class); + } + return jobRepository; + } + + /** + * Lazily retrieves the ReportRepository bean from Spring context. + */ + private ReportRepository getReportRepository() { + if (reportRepository == null && SpringContext.isContextInitialized()) { + reportRepository = SpringContext.getBean(ReportRepository.class); + } + return reportRepository; + } + + /** + * Lazily retrieves the StepReportRepository bean from Spring context. + */ + private StepReportRepository getStepReportRepository() { + if (stepReportRepository == null && SpringContext.isContextInitialized()) { + stepReportRepository = SpringContext.getBean(StepReportRepository.class); + } + return stepReportRepository; + } + + /** + * Checks if the JPA repositories are available (Spring context is initialized). + */ + private boolean isJpaAvailable() { + return SpringContext.isContextInitialized(); + } + public DefaultModelService(StorageService storage, EventsManager eventsManager, NodeType nodeType, String instanceId) { this.storage = storage; @@ -2887,17 +2933,71 @@ public void createOrUpdateJob(Job job) if (job.getInstanceId() == null) { job.setInstanceId(RODAInstanceUtils.getLocalInstanceIdentifier()); } - // create or update job in storage + + // Check if JPA is available and determine persistence strategy + if (isJpaAvailable() && getJobRepository() != null) { + if (Job.isFinalState(job.getState())) { + // Job is in final state - flush to storage and remove from DB + flushJobToStorage(job); + } else { + // Job is running - save to database only + getJobRepository().save(job); + } + } else { + // Fallback to storage-only persistence + String jobAsJson = JsonUtils.getJsonFromObject(job); + StoragePath jobPath = ModelUtils.getJobStoragePath(job.getId()); + storage.updateBinaryContent(jobPath, new StringContentPayload(jobAsJson), false, true, false, null); + } + // index it + notifyJobCreatedOrUpdated(job, false).failOnError(); + } + + /** + * Flushes a job and its reports from the database to the file storage. + * This method is called when a job reaches a final state. + */ + private void flushJobToStorage(Job job) + throws RequestNotValidException, GenericException, NotFoundException, AuthorizationDeniedException { + // Write job to storage String jobAsJson = JsonUtils.getJsonFromObject(job); StoragePath jobPath = ModelUtils.getJobStoragePath(job.getId()); storage.updateBinaryContent(jobPath, new StringContentPayload(jobAsJson), false, true, false, null); - // index it - notifyJobCreatedOrUpdated(job, false).failOnError(); + + // Flush all reports for this job from DB to storage + if (getReportRepository() != null) { + List dbReports = getReportRepository().findByJobId(job.getId()); + for (Report report : dbReports) { + String reportAsJson = JsonUtils.getJsonFromObject(report); + StoragePath reportPath = ModelUtils.getJobReportStoragePath(report.getJobId(), report.getId()); + storage.updateBinaryContent(reportPath, new StringContentPayload(reportAsJson), false, true, false, null); + } + // Delete reports from DB + getReportRepository().deleteByJobId(job.getId()); + } + + // Delete job from DB + JobRepository jobRepo = getJobRepository(); + if (jobRepo != null && jobRepo.existsById(job.getId())) { + jobRepo.deleteById(job.getId()); + } } @Override public Job retrieveJob(String jobId) throws RequestNotValidException, GenericException, NotFoundException, AuthorizationDeniedException { + // Try to fetch from database first (for running jobs) + if (isJpaAvailable()) { + JobRepository jobRepo = getJobRepository(); + if (jobRepo != null) { + Optional dbJob = jobRepo.findById(jobId); + if (dbJob.isPresent()) { + return dbJob.get(); + } + } + } + + // Fallback to storage (for completed/archived jobs) StoragePath jobPath = ModelUtils.getJobStoragePath(jobId); Binary binary = storage.getBinary(jobPath); Job ret; @@ -2913,6 +3013,21 @@ public Job retrieveJob(String jobId) public CloseableIterable> listJobReports(String jobId) throws RequestNotValidException, AuthorizationDeniedException, NotFoundException, GenericException { + // Check if job exists in database (running job) + if (isJpaAvailable()) { + JobRepository jobRepo = getJobRepository(); + ReportRepository reportRepo = getReportRepository(); + if (jobRepo != null && reportRepo != null && jobRepo.existsById(jobId)) { + // Return reports from database + List dbReports = reportRepo.findByJobId(jobId); + List> wrappedReports = dbReports.stream() + .map(OptionalWithCause::of) + .collect(Collectors.toList()); + return CloseableIterables.fromList(wrappedReports); + } + } + + // Fallback to storage final CloseableIterable resourcesIterable = storage .listResourcesUnderContainer(ModelUtils.getJobReportsStoragePath(jobId), false); return ResourceParseUtils.convert(getStorage(), resourcesIterable, Report.class); @@ -2923,18 +3038,65 @@ public void deleteJob(String jobId) throws NotFoundException, GenericException, AuthorizationDeniedException, RequestNotValidException { RodaCoreFactory.checkIfWriteIsAllowedAndIfFalseThrowException(nodeType); - StoragePath jobPath = ModelUtils.getJobStoragePath(jobId); + boolean deletedFromDb = isDeletedFromDb(jobId); - // remove it from storage - storage.deleteResource(jobPath); + // Also try to delete from storage (for archived jobs or cleanup) + try { + StoragePath jobPath = ModelUtils.getJobStoragePath(jobId); + storage.deleteResource(jobPath); + // Also try to delete job reports directory from storage + try { + StoragePath reportsPath = ModelUtils.getJobReportsStoragePath(jobId); + storage.deleteResource(reportsPath); + } catch (NotFoundException e) { + // Reports directory may not exist, ignore + } + } catch (NotFoundException e) { + // If not found in storage and also not deleted from DB, propagate the exception + if (!deletedFromDb) { + throw e; + } + } // remove it from index notifyJobDeleted(jobId).failOnError(); } + private boolean isDeletedFromDb(String jobId) { + boolean deletedFromDb = false; + + // Try to delete from database first (for running jobs) + if (isJpaAvailable()) { + JobRepository jobRepo = getJobRepository(); + ReportRepository reportRepo = getReportRepository(); + if (jobRepo != null && jobRepo.existsById(jobId)) { + // Delete reports from DB + if (reportRepo != null) { + reportRepo.deleteByJobId(jobId); + } + // Delete job from DB + jobRepo.deleteById(jobId); + deletedFromDb = true; + } + } + return deletedFromDb; + } + @Override public Report retrieveJobReport(String jobId, String jobReportId) throws RequestNotValidException, GenericException, NotFoundException, AuthorizationDeniedException { + // Try to fetch from database first (for running jobs) + if (isJpaAvailable()) { + ReportRepository reportRepo = getReportRepository(); + if (reportRepo != null) { + Optional dbReport = reportRepo.findById(jobReportId); + if (dbReport.isPresent()) { + return dbReport.get(); + } + } + } + + // Fallback to storage StoragePath jobReportPath = ModelUtils.getJobReportStoragePath(jobId, jobReportId); Binary binary = storage.getBinary(jobReportPath); Report ret; @@ -2962,14 +3124,39 @@ public void createOrUpdateJobReport(Report jobReport, Job cachedJob) jobReport.setInstanceId(RODAInstanceUtils.getLocalInstanceIdentifier()); - // create job report in storage + // Handle ID change + String newId = IdUtils.getJobReportId(jobReport.getJobId(), jobReport.getSourceObjectId(), + jobReport.getOutcomeObjectId()); + String oldId = null; + if (!newId.equals(jobReport.getId())) { + oldId = jobReport.getId(); + jobReport.setId(newId); + } + + // Check if job exists in database (running job) - use DB for reports + if (isJpaAvailable()) { + JobRepository jobRepo = getJobRepository(); + ReportRepository reportRepo = getReportRepository(); + if (jobRepo != null && reportRepo != null && jobRepo.existsById(jobReport.getJobId())) { + try { + // Delete old report from DB if ID changed + if (oldId != null && reportRepo.existsById(oldId)) { + reportRepo.deleteById(oldId); + notifyJobReportDeleted(oldId); + } + // Save to database + reportRepo.save(jobReport); + // index it + notifyJobReportCreatedOrUpdated(jobReport, cachedJob).failOnError(); + } catch (Exception e) { + LOGGER.error("Error creating/updating job report in database", e); + } + return; + } + } + // Fallback to storage persistence try { - // if job report changed id, set it and remove old report - String newId = IdUtils.getJobReportId(jobReport.getJobId(), jobReport.getSourceObjectId(), - jobReport.getOutcomeObjectId()); - if (!newId.equals(jobReport.getId())) { - String oldId = jobReport.getId(); - jobReport.setId(newId); + if (oldId != null) { storage.deleteResource(ModelUtils.getJobReportStoragePath(jobReport.getJobId(), oldId)); notifyJobReportDeleted(oldId); } @@ -2992,14 +3179,39 @@ public void createOrUpdateJobReport(Report jobReport, IndexedJob indexJob) jobReport.setInstanceId(RODAInstanceUtils.getLocalInstanceIdentifier()); - // create job report in storage + // Handle ID change + String newId = IdUtils.getJobReportId(jobReport.getJobId(), jobReport.getSourceObjectId(), + jobReport.getOutcomeObjectId()); + String oldId = null; + if (!newId.equals(jobReport.getId())) { + oldId = jobReport.getId(); + jobReport.setId(newId); + } + + // Check if job exists in database (running job) - use DB for reports + if (isJpaAvailable()) { + JobRepository jobRepo = getJobRepository(); + ReportRepository reportRepo = getReportRepository(); + if (jobRepo != null && reportRepo != null && jobRepo.existsById(jobReport.getJobId())) { + try { + // Delete old report from DB if ID changed + if (oldId != null && reportRepo.existsById(oldId)) { + reportRepo.deleteById(oldId); + notifyJobReportDeleted(oldId); + } + // Save to database + reportRepo.save(jobReport); + // index it + notifyJobReportCreatedOrUpdated(jobReport, indexJob).failOnError(); + } catch (Exception e) { + LOGGER.error("Error creating/updating job report in database", e); + } + return; + } + } + // Fallback to storage persistence try { - // if job report changed id, set it and remove old report - String newId = IdUtils.getJobReportId(jobReport.getJobId(), jobReport.getSourceObjectId(), - jobReport.getOutcomeObjectId()); - if (!newId.equals(jobReport.getId())) { - String oldId = jobReport.getId(); - jobReport.setId(newId); + if (oldId != null) { storage.deleteResource(ModelUtils.getJobReportStoragePath(jobReport.getJobId(), oldId)); notifyJobReportDeleted(oldId); } @@ -3886,7 +4098,35 @@ public CloseableIterable> list(Cla } else if (DescriptiveMetadata.class.equals(objectClass)) { ret = listDescriptiveMetadata(); } else if (Report.class.equals(objectClass)) { - ret = ResourceParseUtils.convert(getStorage(), listReportResources(), objectClass); + // Include both DB reports (for running jobs) and storage reports (for completed jobs) + CloseableIterable> storageReports = ResourceParseUtils.convert(getStorage(), + listReportResources(), Report.class); + if (isJpaAvailable() && getReportRepository() != null) { + List dbReports = getReportRepository().findAll(); + List> wrappedDbReports = dbReports.stream() + .map(OptionalWithCause::of) + .collect(Collectors.toList()); + CloseableIterable> dbIterable = CloseableIterables.fromList(wrappedDbReports); + ret = CloseableIterables.concat(dbIterable, storageReports); + } else { + ret = storageReports; + } + } else if (Job.class.equals(objectClass)) { + // Include both DB jobs (running) and storage jobs (completed/archived) + StoragePath containerPath = ModelUtils.getContainerPath(objectClass); + final CloseableIterable resourcesIterable = storage.listResourcesUnderContainer(containerPath, false); + CloseableIterable> storageJobs = ResourceParseUtils.convert(getStorage(), + resourcesIterable, Job.class); + if (isJpaAvailable() && getJobRepository() != null) { + List dbJobs = getJobRepository().findAll(); + List> wrappedDbJobs = dbJobs.stream() + .map(OptionalWithCause::of) + .collect(Collectors.toList()); + CloseableIterable> dbIterable = CloseableIterables.fromList(wrappedDbJobs); + ret = CloseableIterables.concat(dbIterable, storageJobs); + } else { + ret = storageJobs; + } } else { StoragePath containerPath = ModelUtils.getContainerPath(objectClass); final CloseableIterable resourcesIterable = storage.listResourcesUnderContainer(containerPath, false); @@ -3921,12 +4161,42 @@ public CloseableIterable> storageReports = ResourceParseUtils.convertLite(getStorage(), + listReportResources(), objectClass); + if (isJpaAvailable() && getReportRepository() != null) { + List dbReports = getReportRepository().findAll(); + List> wrappedDbReports = dbReports.stream() + .map(OptionalWithCause::of) + .collect(Collectors.toList()); + CloseableIterable> dbIterable = LiteRODAObjectFactory + .transformIntoLite(CloseableIterables.fromList(wrappedDbReports)); + ret = CloseableIterables.concat(dbIterable, storageReports); + } else { + ret = storageReports; + } /* * } else if (DisposalConfirmation.class.equals(objectClass)) { ret = * ResourceParseUtils.convertLite(getStorage(), * ResourceListUtils.listDisposalConfirmationResources(storage), objectClass); */ + } else if (Job.class.equals(objectClass)) { + // Include both DB jobs (running) and storage jobs (completed/archived) + StoragePath containerPath = ModelUtils.getContainerPath(objectClass); + final CloseableIterable resourcesIterable = storage.listResourcesUnderContainer(containerPath, false); + CloseableIterable> storageJobs = ResourceParseUtils.convertLite(getStorage(), + resourcesIterable, objectClass); + if (isJpaAvailable() && getJobRepository() != null) { + List dbJobs = getJobRepository().findAll(); + List> wrappedDbJobs = dbJobs.stream() + .map(OptionalWithCause::of) + .collect(Collectors.toList()); + CloseableIterable> dbIterable = LiteRODAObjectFactory + .transformIntoLite(CloseableIterables.fromList(wrappedDbJobs)); + ret = CloseableIterables.concat(dbIterable, storageJobs); + } else { + ret = storageJobs; + } } else { StoragePath containerPath = ModelUtils.getContainerPath(objectClass); final CloseableIterable resourcesIterable = storage.listResourcesUnderContainer(containerPath, false); diff --git a/roda-core/roda-core/src/main/java/org/roda/core/repository/job/JobRepository.java b/roda-core/roda-core/src/main/java/org/roda/core/repository/job/JobRepository.java new file mode 100644 index 0000000000..669a510758 --- /dev/null +++ b/roda-core/roda-core/src/main/java/org/roda/core/repository/job/JobRepository.java @@ -0,0 +1,23 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE file at the root of the source + * tree and available online at + * + * https://github.com/keeps/roda + */ +package org.roda.core.repository.job; + +import org.roda.core.data.v2.jobs.Job; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; + +/** + * JPA Repository for Job entities. Used to store running jobs in the database + * before they are flushed to file storage upon completion. + * + * @author RODA Development Team + */ +@Repository +public interface JobRepository extends JpaRepository { + // Standard JPA methods are inherited from JpaRepository +} diff --git a/roda-core/roda-core/src/main/java/org/roda/core/repository/job/ReportRepository.java b/roda-core/roda-core/src/main/java/org/roda/core/repository/job/ReportRepository.java new file mode 100644 index 0000000000..aa693f105f --- /dev/null +++ b/roda-core/roda-core/src/main/java/org/roda/core/repository/job/ReportRepository.java @@ -0,0 +1,60 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE file at the root of the source + * tree and available online at + * + * https://github.com/keeps/roda + */ +package org.roda.core.repository.job; + +import java.util.List; +import java.util.Optional; + +import org.roda.core.data.v2.jobs.Report; +import org.springframework.data.jpa.repository.EntityGraph; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; +import org.springframework.transaction.annotation.Transactional; + +/** + * JPA Repository for Report entities. Used to store job reports in the database + * while the associated job is running, before they are flushed to file storage. + * + * @author RODA Development Team + */ +@Repository +public interface ReportRepository extends JpaRepository { + + /** + * Find a report by its ID, eagerly fetching step reports to avoid + * LazyInitializationException when accessed outside a transaction. + * + * @param id + * the report ID + * @return optional containing the report if found + */ + @Override + @EntityGraph(attributePaths = "stepReports") + Optional findById(String id); + + /** + * Find all reports associated with a given job ID, eagerly fetching step + * reports to avoid LazyInitializationException when accessed outside a + * transaction. + * + * @param jobId + * the job ID to search for + * @return list of reports for the specified job + */ + @EntityGraph(attributePaths = "stepReports") + List findByJobId(String jobId); + + /** + * Delete all reports associated with a given job ID. + * + * @param jobId + * the job ID whose reports should be deleted + */ + @Transactional + void deleteByJobId(String jobId); +} diff --git a/roda-core/roda-core/src/main/java/org/roda/core/repository/job/StepReportRepository.java b/roda-core/roda-core/src/main/java/org/roda/core/repository/job/StepReportRepository.java new file mode 100644 index 0000000000..1b751ce4de --- /dev/null +++ b/roda-core/roda-core/src/main/java/org/roda/core/repository/job/StepReportRepository.java @@ -0,0 +1,43 @@ +/** + * The contents of this file are subject to the license and copyright + * detailed in the LICENSE file at the root of the source + * tree and available online at + * + * https://github.com/keeps/roda + */ +package org.roda.core.repository.job; + +import java.util.List; + +import org.roda.core.data.v2.jobs.StepReport; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.stereotype.Repository; +import org.springframework.transaction.annotation.Transactional; + +/** + * JPA Repository for StepReport entities. Used to store step reports in the database + * while the associated job is running, before they are flushed to file storage. + * + * @author RODA Development Team + */ +@Repository +public interface StepReportRepository extends JpaRepository { + + /** + * Find all step reports associated with a given parent report ID. + * + * @param parentReportId + * the parent report ID to search for + * @return list of step reports for the specified parent report, ordered by step order + */ + List findByParentReportIdOrderByStepOrderAsc(String parentReportId); + + /** + * Delete all step reports associated with a given parent report ID. + * + * @param parentReportId + * the parent report ID whose step reports should be deleted + */ + @Transactional + void deleteByParentReportId(String parentReportId); +} diff --git a/roda-ui/roda-wui/pom.xml b/roda-ui/roda-wui/pom.xml index 5986e6b065..5240e10bd2 100644 --- a/roda-ui/roda-wui/pom.xml +++ b/roda-ui/roda-wui/pom.xml @@ -124,8 +124,6 @@ -Droda.home=${env.HOME}/.roda_local -Droda.environment.collect.version=false -Dgwt.codeServerPort=9876 -Xdebug - - -Dorg.springframework.boot.logging.LoggingSystem=none -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=*:5007 --add-opens java.base/java.util=ALL-UNNAMED --add-opens java.base/java.lang=ALL-UNNAMED @@ -182,8 +180,6 @@ -Droda.home=${env.HOME}/.roda_central -Droda.environment.collect.version=false -Dgwt.codeServerPort=9876 -Xdebug - - -Dorg.springframework.boot.logging.LoggingSystem=none -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=*:5006 --add-opens java.base/java.util=ALL-UNNAMED --add-opens java.base/java.lang=ALL-UNNAMED @@ -238,9 +234,6 @@ -Dgwt.codeServerPort=9876 -Xdebug - - -Dorg.springframework.boot.logging.LoggingSystem=none - -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=*:5005 -Droda.environment.collect.version=false --add-opens java.base/java.util=ALL-UNNAMED diff --git a/roda-ui/roda-wui/src/main/java/org/roda/wui/RODA.java b/roda-ui/roda-wui/src/main/java/org/roda/wui/RODA.java index 43f2d6f28b..055ddbdfe8 100644 --- a/roda-ui/roda-wui/src/main/java/org/roda/wui/RODA.java +++ b/roda-ui/roda-wui/src/main/java/org/roda/wui/RODA.java @@ -23,7 +23,7 @@ UserDetailsServiceAutoConfiguration.class}) @ComponentScan(basePackages = {"org.roda.*"}) @EnableJpaRepositories(basePackages = "org.roda.core.repository") -@EntityScan(basePackages = "org.roda.core.entity") +@EntityScan(basePackages = {"org.roda.core.entity", "org.roda.core.data.v2.jobs"}) @ServletComponentScan @EnableScheduling public class RODA { diff --git a/roda-ui/roda-wui/src/main/resources/config/logback_wui.xml b/roda-ui/roda-wui/src/main/resources/config/logback_wui.xml index b237b08d40..b566f055bf 100644 --- a/roda-ui/roda-wui/src/main/resources/config/logback_wui.xml +++ b/roda-ui/roda-wui/src/main/resources/config/logback_wui.xml @@ -59,4 +59,11 @@ + + + + + + + \ No newline at end of file