fetch new values from config

Signed-off-by: Adrian Edwards <adredwar@redhat.com>

Author: MoralCode

augur/application/cli/backend.py

@@ -87,7 +87,10 @@ def start(ctx, disable_collection, development, pidfile, port):
     if disable_collection:
         os.environ["AUGUR_DISABLE_COLLECTION"] = "1"
 
-    worker_vmem_cap = get_value("Celery", 'worker_process_vmem_cap')
+    core_worker_count = get_value("Celery", 'core_worker_count')
+    secondary_worker_count = get_value("Celery", 'secondary_worker_count')
+    facade_worker_count = get_value("Celery", 'facade_worker_count')
+
 
     # create rabbit messages so if it failed on shutdown the queues are clean
     cleanup_collection_status_and_rabbit(logger, ctx.obj.engine)
@@ -119,7 +122,7 @@ def start(ctx, disable_collection, development, pidfile, port):
     logger.info(f'Augur is running at: {"http" if development else "https"}://{host}:{port}')
     logger.info(f"The API is available at '{api_response.json()['route']}'")
 
-    processes = start_celery_worker_processes(float(worker_vmem_cap), disable_collection)
+    processes = start_celery_worker_processes((core_worker_count, secondary_worker_count, facade_worker_count), disable_collection)
 
     celery_beat_schedule_db = os.getenv("CELERYBEAT_SCHEDULE_DB", "celerybeat-schedule.db")
     if os.path.exists(celery_beat_schedule_db):

augur/application/cli/collection.py

@@ -78,9 +78,11 @@ def start(ctx, development):
         os.environ["AUGUR_DEV"] = "1"
         logger.info("Starting in development mode")
 
-    worker_vmem_cap = get_value("Celery", 'worker_process_vmem_cap')
+    core_worker_count = get_value("Celery", 'core_worker_count')
+    secondary_worker_count = get_value("Celery", 'secondary_worker_count')
+    facade_worker_count = get_value("Celery", 'facade_worker_count')
 
-    process_list = start_celery_collection_processes(float(worker_vmem_cap))
+    process_list = start_celery_collection_processes((core_worker_count, secondary_worker_count, facade_worker_count))
 
     if os.path.exists("celerybeat-schedule.db"):
             logger.info("Deleting old task schedule")

augur/application/config.py

@@ -69,7 +69,9 @@ def get_development_flag():
                 "log_level": "INFO",
             },
             "Celery": {
-                "worker_process_vmem_cap": 0.25,
+                "core_worker_count": 5,
+                "secondary_worker_count": 5,
+                "facade_worker_count": 5,
                 "refresh_materialized_views_interval_in_days": 1
             },
             "Redis": {

docs/source/getting-started/collecting-data.rst

@@ -88,8 +88,11 @@ Celery Configuration
 
 **We strongly recommend leaving the default celery blocks generated by the installation process, but if you would like to know more, or fine-tune them to your needs, read on.**
 
-The celery monitor is responsible for generating the tasks that will tell the other worker processes what data to collect, and how. The ``Celery`` block has 2 keys; one for memory cap and one for materialized views interval.
-- ``worker_process_vmem_cap``, float between zero and one that determines the maximum percentage of total memory to use for worker processes
+The celery monitor is responsible for generating the tasks that will tell the other worker processes what data to collect, and how. The ``Celery`` block has several keys:
+
+- ``core_worker_count``, the number of workers to spawn to run the core tasks.
+- ``secondary_worker_count``, the number of workers to spawn to run the secondary tasks.
+- ``facade_worker_count``, the number of workers to spawn to run the facade tasks.
 
 - ``refresh_materialized_views_interval_in_days``, number of days to wait between refreshes of materialized views.