PR: Add a tool to import HG surveys

sardes/api/database_accessor.py

@@ -47,6 +47,9 @@ def _transaction_wrapper(self, func, mode, *args, **kwargs):
     self.begin_transaction()
     try:
         results = func(self, *args, **kwargs)
+        # TODO: I am not sure we should auto commit transaction for every
+        # read. What if we want to do a read in-between two write within
+        # the same transaction?
         if mode == 'read' or kwargs.get('auto_commit', True):
             self.commit_transaction()
     finally:

sardes/api/database_model.py

@@ -534,7 +534,8 @@ def __delitem__(self, key):
                 name='hg_param_name',
                 dtype='str',
                 desc=("The name of the hydrogeochemical parameter."
-                      "Used when importing results from a lab report."),
+                      "Used when exporting results to a water "
+                      "quality report."),
             ),
             Column(
                 name='hg_param_regex',

sardes/api/plugins.py

@@ -322,9 +322,11 @@ def __init__(self, parent=None):
         # to be displayed in Sardes main window for this plugin.
         self.dockwindow = None
 
+        # Plugin setup.
         self.setup_plugin()
         self.pane_widget = self.create_pane_widget()
         self.mainwindow_toolbars = self.create_mainwindow_toolbars()
+        self.__post_init__()
 
     def main_option_actions(self):
         return []
@@ -499,6 +501,10 @@ def get_plugin_icon(cls):
         """
         return get_icon('master')
 
+    def __post_init__(self):
+        """A method that is called at the end of SardesPlugin constructor."""
+        pass
+
     def setup_plugin(self):
         pass
 

sardes/config/icons.py

@@ -156,6 +156,9 @@
     'import_clipboard': [
         ('mdi.clipboard-plus-outline',),
         {'color': ICON_COLOR, 'scale_factor': 1.2}],
+    'import_geochemistry': [
+        ('mdi.flask-plus-outline',),
+        {'color': ICON_COLOR, 'scale_factor': 1.2}],
     'information': [
         ('mdi.information-outline',),
         {'color': ICON_COLOR, 'scale_factor': 1.3}],

sardes/database/accessors/accessor_errors.py

@@ -11,6 +11,13 @@
 from sardes.config.locale import _
 
 
+class ImportHGSurveysError(Exception):
+    def __init__(self, message: str, code: int):
+        super().__init__(message)
+        self.message = message
+        self.code = code
+
+
 class SardesVersionError(Exception):
     """
     Exception that should be raised when the version of Sardes is outdated

sardes/database/accessors/tests/conftest.py

@@ -285,10 +285,10 @@ def pump_types():
 @pytest.fixture
 def hg_params():
     df = pd.DataFrame(
-        data=[['PARAM#1', 'Name param #1', 'PARAM#1 Regex', '111-0-11'],
-              ['PARAM#2', 'Name param #2', 'PARAM#2 Regex', '22-0-23'],
-              ['PARAM#3', 'Name param #3', 'PARAM#3 Regex', '34-5-12'],
-              ['PARAM#4', 'Name param #4', 'PARAM#4 Regex', '73-2-9']],
+        data=[['PARAM#1', 'Name param #1', 'PARAM(.*)?1', '111-0-11'],
+              ['PARAM#2', 'Name param #2', 'PARAM(.*)?2', '22-0-23'],
+              ['PARAM#3', 'Name param #3', 'PARAM(.*)?3', '34-5-12'],
+              ['PARAM#4', 'Name param #4', 'PARAM(.*)?4', '73-2-9']],
         index=[1, 2, 3, 4],
         columns=['hg_param_code', 'hg_param_name', 'hg_param_regex',
                  'cas_registry_number']

sardes/database/database_manager.py

@@ -31,6 +31,7 @@
 from sardes.config.ospath import get_documents_logo_filename
 from sardes.config.main import CONF
 from sardes.database.accessors.accessor_helpers import create_empty_readings
+from sardes.database.accessors.accessor_errors import ImportHGSurveysError
 from sardes.tools.hydrographs import HydrographCanvas
 from sardes.tools.save2excel import _save_reading_data_to_xlsx
 from sardes.tools.waterquality import _save_hg_data_to_xlsx
@@ -456,7 +457,7 @@ def _get_sonde_installation_info(self, sonde_serial_no, date_time):
 
         return sonde_install,
 
-    # ---- Publish Network Data
+    # ---- Complex operations
     def _publish_to_kml(self, kml_filename, iri_data=None, iri_logs=None,
                         iri_graphs=None, iri_quality=None):
         """
@@ -709,6 +710,93 @@ def _publish_to_kml(self, kml_filename, iri_data=None, iri_logs=None,
         kml.save(kml_filename)
         return True,
 
+    def _add_hg_survey_data(self, imported_survey_data: dict(dict)):
+        """
+        Add HG survey data imported from a XLSX file.
+        """
+        from sardes.plugins.hydrogeochemistry.hgsurveys import (
+            format_hg_survey_imported_data,
+            format_purge_imported_data,
+            format_params_data_imported_data
+            )
+
+        hg_surveys_data, = self._get('hg_surveys')
+        stations_data, = self._get('observation_wells_data')
+        hg_sampling_methods_data, = self._get('hg_sampling_methods')
+        pump_types_data, = self._get('pump_types')
+        hg_params_data, = self._get('hg_params')
+        measurement_units_data, = self._get('measurement_units')
+
+        new_hg_surveys = []
+        new_purges = []
+        new_hg_param_vals = []
+
+        for sheet_name, sheet_data in imported_survey_data.items():
+            try:
+                new_hg_surveys.append(
+                    format_hg_survey_imported_data(
+                        sheet_name,
+                        sheet_data['hg_surveys_data'],
+                        hg_surveys_data,
+                        stations_data,
+                        hg_sampling_methods_data
+                        ))
+                new_purges.extend(
+                    format_purge_imported_data(
+                        sheet_name,
+                        sheet_data['purges_data'],
+                        pump_types_data
+                        ))
+                new_hg_param_vals.extend(
+                    format_params_data_imported_data(
+                        sheet_name,
+                        sheet_data['hg_param_values_data'],
+                        hg_params_data,
+                        measurement_units_data
+                        ))
+            except ImportHGSurveysError as e:
+                return e,
+
+        # Clear the cache.
+        for name in ['hg_surveys', 'purges', 'hg_param_values']:
+            if name in self._cache:
+                del self._cache[name]
+
+        # We first add the new HG surveys to the database.
+        indexes = self.db_accessor.add(
+            'hg_surveys', new_hg_surveys, auto_commit=False)
+
+        # Substitute the right HG survey id for the purges and HG param values.
+        sheet_names = list(imported_survey_data.keys())
+        indexes_map = {
+            sheet_name: hg_survey_id for
+            sheet_name, hg_survey_id in
+            zip(sheet_names, indexes)
+            }
+
+        for new_purge in new_purges:
+            new_purge['hg_survey_id'] = indexes_map[
+                new_purge['hg_survey_id']]
+        for new_hg_param_val in new_hg_param_vals:
+            new_hg_param_val['hg_survey_id'] = indexes_map[
+                new_hg_param_val['hg_survey_id']]
+
+        # Add the new purges and HG parameter values to the database.
+        self.db_accessor.add(
+            'purges', new_purges, auto_commit=False)
+        self.db_accessor.add(
+            'hg_param_values', new_hg_param_vals, auto_commit=True)
+
+        if len(imported_survey_data) == 1:
+            message = _(
+                "1 new survey has been imported into the database."
+                )
+        else:
+            message = _(
+                "{} new surveys have been imported into the database."
+                ).format(len(imported_survey_data))
+        return message,
+
 
 class DatabaseConnectionManager(TaskManagerBase):
     sig_database_connected = Signal(object, object)
@@ -1026,7 +1114,7 @@ def _handle_run_tasks_finished(self):
                 list(self._tseries_data_changed))
             self._tseries_data_changed = set()
 
-    # ---- Publish Network Data
+    # ---- Other
     def publish_to_kml(self, filename, iri_data=None, iri_logs=None,
                        iri_graphs=None, iri_quality=None, callback=None,
                        postpone_exec=False):
@@ -1038,6 +1126,19 @@ def publish_to_kml(self, filename, iri_data=None, iri_logs=None,
         if not postpone_exec:
             self.run_tasks()
 
+    def add_hg_survey_data(self, imported_survey_data: dict(dict),
+                           callback: Callable = None,
+                           postpone_exec: bool = False):
+        """
+        Add HG survey data imported from a XLSX file.
+        """
+        self._data_changed.add('hg_surveys')
+        self._data_changed.add('purges')
+        self._data_changed.add('hg_param_values')
+        self.add_task('add_hg_survey_data', callback, imported_survey_data)
+        if not postpone_exec:
+            self.run_tasks()
+
 
 if __name__ == '__main__':
     from sardes.database.accessors import DatabaseAccessorSardesLite

sardes/plugins/conftest.py

@@ -8,5 +8,24 @@
 # -----------------------------------------------------------------------------
 
 
+# ---- Standard imports
+import os
+import os.path as osp
+os.environ['SARDES_PYTEST'] = 'True'
+
+# ---- Third party imports
 import pytest
+
+# ---- Local imports
+from sardes.database.accessors import DatabaseAccessorSardesLite
 from sardes.database.accessors.tests.conftest import *
+
+
+@pytest.fixture
+def dbaccessor(tmp_path, database_filler):
+    dbaccessor = DatabaseAccessorSardesLite(
+        osp.join(tmp_path, 'sqlite_database_test.db'))
+    dbaccessor.init_database()
+    database_filler(dbaccessor)
+
+    return dbaccessor