Deduplicate/lib get_section - and some other config improvements

augur/api/routes/dei.py

@@ -69,7 +69,7 @@ def dei_track_repo(application: ClientApplication):
         "repo_id": repo_id
     }
 
-    enabled_phase_names = get_enabled_phase_names_from_config()
+    enabled_phase_names = get_enabled_phase_names_from_config_session(session, logger)
 
     #Primary collection hook.
     primary_enabled_phases = []

augur/application/config.py

@@ -38,7 +38,6 @@ def get_development_flag():
                 "github": "<gh_api_key>",
                 "gitlab": "<gl_api_key>"
             },
-            #TODO: a lot of these are deprecated.
             "Facade": {
                 "check_updates": 1,
                 "create_xlsx_summary_files": 1,

augur/application/db/lib.py

@@ -4,7 +4,7 @@
 import logging
 import sqlalchemy as s
 from sqlalchemy import func 
 from sqlalchemy.exc import DataError
 from sqlalchemy.dialects import postgresql
 from sqlalchemy.exc import OperationalError
 from psycopg2.errors import DeadlockDetected
@@ -19,34 +19,6 @@
 logger = logging.getLogger("db_lib")
 
 
-def get_section(section_name) -> dict:
-    """Get a section of data from the config.
-
-    Args:
-        section_name: The name of the section being retrieved
-
-    Returns:
-        The section data as a dict
-    """
-    with get_session() as session:
-
-        query = session.query(Config).filter_by(section_name=section_name)
-        section_data = execute_session_query(query, 'all')
-
-        section_dict = {}
-        for setting in section_data:
-            setting_dict = setting.__dict__
-
-            setting_dict = convert_type_of_value(setting_dict, logger)
-
-            setting_name = setting_dict["setting_name"]
-            setting_value = setting_dict["value"]
-
-            section_dict[setting_name] = setting_value
-
-        return section_dict
-
-
 def get_value(section_name: str, setting_name: str) -> Optional[Any]:
     """Get the value of a setting from the config.
 
@@ -167,7 +139,7 @@
 
     try:
         working_commits = fetchall_data_from_sql_text(query)
     except:
         working_commits = []
 
     return working_commits
@@ -183,7 +155,7 @@
 
     try:
         missing_commit_hashes = fetchall_data_from_sql_text(fetch_missing_hashes_sql)
     except:
         missing_commit_hashes = []
 
     return missing_commit_hashes
@@ -203,7 +175,7 @@
         return session.query(CollectionStatus).filter(getattr(CollectionStatus,f"{collection_type}_status" ) == CollectionState.COLLECTING.value).count()
 
 
 def facade_bulk_insert_commits(logger, records):
 
     with get_session() as session:
 
@@ -272,7 +244,7 @@
             else:
                 raise e
 
 def batch_insert_contributors(logger, data: Union[List[dict], dict], batch_size = 1000) -> Optional[List[dict]]:
 
     for i in range(0, len(data), batch_size):
         batch = data[i:i + batch_size]
@@ -283,7 +255,7 @@
 
 
 
 def bulk_insert_dicts(logger, data_input: Union[List[dict], dict], table, natural_keys: List[str], return_columns: Optional[List[str]] = None, string_fields: Optional[List[str]] = None, on_conflict_update:bool = True) -> Optional[List[dict]]:
 
     if isinstance(data_input, list) is False:
 

augur/tasks/git/util/facade_worker/facade_worker/config.py

@@ -35,8 +35,8 @@
 from psycopg2.errors import DeadlockDetected
 
 from augur.application.db.session import DatabaseSession
+from augur.application.config import AugurConfig
 from augur.application.db.lib import execute_sql
-from augur.application.db.lib import get_section
 from logging import Logger
 
 logger = logging.getLogger(__name__)
@@ -110,7 +110,10 @@ def __init__(self,logger: Logger):
 
         self.logger = logger
 
-        worker_options = get_section("Facade")
+        with DatabaseSession(logger, engine) as session:
+            config = AugurConfig(logger, session)
+
+            worker_options = config.get_section("Facade")
 
         self.limited_run = worker_options["limited_run"]
         self.delete_marked_repos = worker_options["delete_marked_repos"]
@@ -251,256 +254,3 @@ def insert_or_update_data(self, query, **bind_args)-> None:
             return
     def inc_repos_processed(self):
         self.repos_processed += 1
-
-#    def get_last_collected_commit_date(self,repo_id):
-#        commit_date_query = s.sql.text("""
-#        SELECT cmt_committer_timestamp FROM commits 
-#        WHERE repo_id=:repo_id 
-#        ORDER BY data_collection_date DESC
-#        LIMIT 1;
-#        """).bindparams(repo_id=repo_id)
-#
-#        result = execute_sql(commit_date_query).fetchone()
-#        return result[0]
-
-"""
-class FacadeConfig:
-    \"""Legacy facade config that holds facade's database functionality
-
-        This is mainly for compatibility with older functions from legacy facade.
-
-        Initializes database when it encounters a database exception
-
-    Attributes:
-        repos_processed (int): Counter for how many repos have been analyzed
-        cursor (psycopg2.extensions.cursor): database cursor for legacy facade.
-        logger (Logger): logger object inherited from the session object
-        db (psycopg2.extensions.connection): database connection object for legacy facade.
-        tool_source (str): String marking the source of data as from facade.
-        data_source (str): String indicating that facade gets data from git
-        tool_version (str): Facade version
-        worker_options (dict): Config options for facade.
-        log_level (str): Keyword indicating level of logging for legacy facade.
-    \"""
-    def __init__(self, logger: Logger):
-        self.repos_processed = 0
-        self.cursor = None
-        self.logger = logger
-
-        self.db = None
-
-        #init db first thing 
-        db_credentials = get_database_args_from_env()
-
-        # Set up the database
-        db_user = db_credentials["db_user"]
-        db_pass = db_credentials["db_pass"]
-        db_name = db_credentials["db_name"]
-        db_host = db_credentials["db_host"]
-        db_port = db_credentials["db_port"]
-        db_user_people = db_user
-        db_pass_people = db_pass
-        db_name_people = db_name
-        db_host_people = db_host
-        db_port_people = db_port
-        # Open a general-purpose connection
-        db,cursor = self.database_connection(
-            db_host,
-            db_user,
-            db_pass,
-            db_name,
-            db_port, False, False)
-
-        #worker_options = read_config("Workers", "facade_worker", None, None)
-
-        with DatabaseSession(logger, engine) as session:
-            config = AugurConfig(logger, session)
-            worker_options = config.get_section("Facade")
-
-        if 'repo_directory' in worker_options:
-            self.repo_base_directory = worker_options['repo_directory']
-        else:
-            self.log_activity('Error',"Please specify a \'repo_directory\' parameter"
-                " in your \'Workers\' -> \'facade_worker\' object in your config "
-                "to the directory in which you want to clone repos. Exiting...")
-            sys.exit(1)
-
-        self.tool_source = '\'Facade \''
-        self.tool_version = '\'1.3.0\''
-        self.data_source = '\'Git Log\''
-
-        self.worker_options = worker_options
-
-        # Figure out how much we're going to log
-        #logging.basicConfig(filename='worker_{}.log'.format(worker_options['port']), filemode='w', level=logging.INFO)
-        self.log_level = None #self.get_setting('log_level')
-
-
-    #### Database update functions ####
-
-    def increment_db(self, version):
-
-        # Helper function to increment the database number
-
-        increment_db = ("INSERT INTO settings (setting,value) "
-            "VALUES ('database_version',%s)")
-        self.cursor.execute(increment_db, (version, ))
-        db.commit()
-
-        print("Database updated to version: %s" % version)
-
-    def update_db(self, version):
-
-        # This function should incrementally step any version of the database up to
-        # the current schema. After executing the database operations, call
-        # increment_db to bring it up to the version with which it is now compliant.
-
-        print("Legacy Facade Block for DB UPDATE. No longer used. ")
-
-        print("No further database updates.\n")
-
-    def migrate_database_config(self):
-
-    # Since we're changing the way we store database credentials, we need a way to
-    # transparently migrate anybody who was using the old file. Someday after a long
-    # while this can disappear.
-
-        try:
-            # If the old database config was found, write a new config
-            imp.find_module('db')
-
-            db_config = configparser.ConfigParser()
-
-            from db import db_user,db_pass,db_name,db_host
-            from db import db_user_people,db_pass_people,db_name_people,db_host_people
-
-            db_config.add_section('main_database')
-            db_config.set('main_database','user',db_user)
-            db_config.set('main_database','pass',db_pass)
-            db_config.set('main_database','name',db_name)
-            db_config.set('main_database','host',db_host)
-
-            db_config.add_section('people_database')
-            db_config.set('people_database','user',db_user_people)
-            db_config.set('people_database','pass',db_pass_people)
-            db_config.set('people_database','name',db_name_people)
-            db_config.set('people_database','host',db_host_people)
-
-            with open('db.cfg','w') as db_file:
-                db_config.write(db_file)
-
-            print("Migrated old style config file to new.")
-        except:
-            # If nothing is found, the user probably hasn't run setup yet.
-            sys.exit("Can't find database config. Have you run setup.py?")
-
-        try:
-            os.remove('db.py')
-            os.remove('db.pyc')
-            print("Removed unneeded config files")
-        except:
-            print("Attempted to remove unneeded config files")
-
-        return db_user,db_pass,db_name,db_host,db_user_people,db_pass_people,db_name_people,db_host_people
-
-    #### Global helper functions ####
-
-    def database_connection(self, db_host,db_user,db_pass,db_name, db_port, people, multi_threaded_connection):
-
-    # Return a database connection based upon which interpreter we're using. CPython
-    # can use any database connection, although MySQLdb is preferred over pymysql
-    # for performance reasons. However, PyPy can't use MySQLdb at this point,
-    # instead requiring a pure python MySQL client. This function returns a database
-    # connection that should provide maximum performance depending upon the
-    # interpreter in use.
-
-    ##TODO: Postgres connections as we make them ARE threadsafe. We *could* refactor this accordingly: https://www.psycopg.org/docs/connection.html #noturgent
-
-
-        # if platform.python_implementation() == 'PyPy':
-        db_schema = 'augur_data'
-        db = psycopg2.connect(
-            host = db_host,
-            user = db_user,
-            password = db_pass,
-            database = db_name,
-            # charset = 'utf8mb4',
-            port = db_port,
-            options=f'-c search_path={db_schema}',
-            connect_timeout = 31536000,)
-
-        cursor = db.cursor()#pymysql.cursors.DictCursor)
-
-
-        self.cursor = cursor
-        self.db = db
-
-        # Figure out how much we're going to log
-        #self.log_level = self.get_setting('log_level')
-        #Not getting debug logging for some reason.
-        self.log_level = 'Debug'
-        return db, cursor
-
-    def get_setting(self, setting):
-
-    # Get a setting from the database
-
-        query = (\"""SELECT value FROM settings WHERE setting=%s ORDER BY
-            last_modified DESC LIMIT 1\""")
-        self.cursor.execute(query, (setting, ))
-        # print(type(self.cursor.fetchone()))
-        return self.cursor.fetchone()[0]#["value"]
-
-    def update_status(self, status):
-
-    # Update the status displayed in the UI
-
-        query = ("UPDATE settings SET value=%s WHERE setting='utility_status'")
-        self.cursor.execute(query, (status, ))
-        self.db.commit()
-
-    def log_activity(self, level, status):
-
-    # Log an activity based upon urgency and user's preference.  If the log level is
-    # "Debug", then just print it and don't save it in the database.
-
-        log_options = ('Error','Quiet','Info','Verbose','Debug')
-        self.logger.info("* %s\n" % status)
-        if self.log_level == 'Debug' and level == 'Debug':
-            return
-
-        #if log_options.index(level) <= log_options.index(self.log_level):
-        query = ("INSERT INTO utility_log (level,status) VALUES (%s,%s)")
-        try:
-            self.cursor.execute(query, (level, status))
-            self.db.commit()
-        except Exception as e:
-            self.logger.info('Error encountered: {}\n'.format(e))
-
-            db_credentials = get_database_args_from_env()
-
-            # Set up the database
-            db_user = db_credentials["db_user"]
-            db_pass = db_credentials["db_pass"]
-            db_name = db_credentials["db_name"]
-            db_host = db_credentials["db_host"]
-            db_port = db_credentials["db_port"]
-            db_user_people = db_user
-            db_pass_people = db_pass
-            db_name_people = db_name
-            db_host_people = db_host
-            db_port_people = db_port
-            # Open a general-purpose connection
-            db,cursor = self.database_connection(
-                db_host,
-                db_user,
-                db_pass,
-                db_name,
-                db_port, False, False)
-            self.cursor.execute(query, (level, status))
-            self.db.commit()
-
-
-
-
-"""

augur/tasks/start_tasks.py

@@ -140,7 +140,7 @@
 
     logger.info("Executing non-repo domain tasks")
 
-    enabled_phase_names = get_enabled_phase_names_from_config()
+    enabled_phase_names = get_enabled_phase_names_from_config(engine, logger)
 
     enabled_tasks = []
 
@@ -245,7 +245,7 @@
 
 
     #Get list of enabled phases 
-    enabled_phase_names = get_enabled_phase_names_from_config()
+    enabled_phase_names = get_enabled_phase_names_from_config(engine, logger)
 
     enabled_collection_hooks = []
 
@@ -295,7 +295,7 @@
             status = repo.collection_status[0]
             raw_count = status.issue_pr_sum
 
             issue_pr_task_update_weight_util([int(raw_count)],repo_git=repo_git,session=session)
 
         facade_not_pending = CollectionStatus.facade_status != CollectionState.PENDING.value
         facade_not_failed = CollectionStatus.facade_status != CollectionState.FAILED_CLONE.value

augur/tasks/util/collection_util.py

@@ -10,12 +10,13 @@
 from augur.tasks.init.celery_app import celery_app as celery
 from augur.application.db.models import CollectionStatus, Repo
 from augur.application.db.util import execute_session_query
-from augur.application.db.lib import get_section
 from augur.tasks.github.util.util import get_repo_weight_core, get_repo_weight_by_issue
 from augur.application.db import get_engine
 from augur.application.db.lib import execute_sql, get_session, get_active_repo_count, get_repo_by_repo_git
 from augur.tasks.util.worker_util import calculate_date_weight_from_timestamps
 from augur.tasks.util.collection_state import CollectionState
+from augur.application.db.session import DatabaseSession
+from augur.application.config import AugurConfig
 
 
 def get_list_of_all_users():
@@ -204,9 +205,19 @@
     return valid_repo_git_list
 
 
-def get_enabled_phase_names_from_config():
+def get_enabled_phase_names_from_config(engine, logger):
+    with DatabaseSession(logger, engine) as session:
+        return get_enabled_phase_names_from_config_session(session, logger)
 
-    phase_options = get_section("Task_Routine")
+
+def get_enabled_phase_names_from_config_session(session, logger):
+
+    config = AugurConfig(logger, session)
+    return get_enabled_phase_names_from_config_object(config)
+
+
+def get_enabled_phase_names_from_config_object(config):
+    phase_options = config.get_section("Task_Routine")
 
     #Get list of enabled phases 
     enabled_phase_names = [name for name, phase in phase_options.items() if phase == 1]
@@ -261,7 +272,7 @@
 
         try:
             collectionRecord = execute_session_query(query,'one')
         except:
             #Exit if we can't find the record.
             return
 
@@ -333,7 +344,7 @@
         raw_count = status.issue_pr_sum
 
         #Update the values for core and secondary weight
         issue_pr_task_update_weight_util([int(raw_count)],repo_git=repo_git,session=session)
 
 #Update the existing core and secondary weights as well as the raw sum of issues and prs
 def update_issue_pr_weights(logger,session,repo_git,raw_sum):
@@ -400,7 +411,7 @@
         status = repo.collection_status[0]
         raw_count = status.issue_pr_sum
 
         issue_pr_task_update_weight_util([int(raw_count)],repo_git=repo_git,session=session)
 
 #Get the weight for each repo for the secondary collection hook.
 def get_repo_weight_secondary(logger,repo_git):