Merge pull request #12 from NHSDigital/midyear_changes_2425_LC

2024-25 mid year changes from GitLab to GitHub.

Author: LauraCorbett

.gitignore

@@ -5,6 +5,7 @@
 *.ipynb
 *.xlsx
 *.xls
+Outputs/
 
 
 

CHANGELOG.txt

@@ -1,5 +1,14 @@
 NCDes new CQRS data update.
 
+Changelog V1.2
+------------------------------------------------------------------
+Updated metadata files
+Updates made to the run_ncdes.bat file for .venv creation and update. 
+Replacement of the enviroment.yml file with a requirements.txt file.
+measure_and_indicator_mappings() function updated to account for removal of CAN02 and replaced with CAN04, in line with the 24-25 Mid year changes.
+Addition of git project supporting documents added to the "supporting_docs" folder.
+
+
 Changelog V1.1
 ------------------------------------------------------------------
 Updated metadata files
@@ -30,3 +39,4 @@ Added three files in root_directory:
 input/CQRS_files/NCD_CQRS_Metadata_Mapping.csv - COntains a mapping table to fix field names
 input/CQRS_files/NCD_metadata.csv - Contains a table to produce indicator and measure mappings
 output/CQRS_omitted_tracker.csv - Contains omitted practices for certain rules (does not count PCN codes)
+

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2022, Crown Copyright NHS Digital
+Copyright (c) 2024, Crown Copyright NHS England.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -1,10 +1,10 @@
-> Warning: This is the README for the publically accessible version of the NCDes package. If you are an analyst please don't use the below instructions to run the publication process.
+> Warning: This is the README for the publically accessible version of the NCDes package. If you are an internal analyst please don't use the below instructions to run the publication process.
 
 <p>&nbsp;</p>
 
-Repository owner: Primary Care Domain Analytical Team
+Repository owner: General Practice Data, Extracts, Specifications and Analysis (GPDESA) Team
 
-Email: primarycare.domain@nhs.net
+Email: gpses@nhs.net
 
 To contact us raise an issue on Github or via email and we will respond promptly.
 
@@ -61,7 +61,7 @@ root
 ## Instructions for publication production
 After the above set up steps have been completed you can follow the below instructions to create the publication. Please note that you will not be able to run the code as this requires access to a private server. The data on the private server contains reference data that is used for mapping purposes. The reference tables used contain data from the [epraccur file](https://digital.nhs.uk/services/organisation-data-service/file-downloads/gp-and-gp-practice-related-data) and the [ONS code history database](https://www.ons.gov.uk/methodology/geography/geographicalproducts/namescodesandlookups/codehistorydatabasechd)
 
-1) In the config file edit the root directory value so that it matches the root of the directory that you set up. Make use of escape characters and end path with a double "\\\\" e.g. "\\\\\\\example\\\root\\\directory\\\\".
+1) Review the config file and edit the paramaters as necessary e.g. edit the root directory value so that it matches the root of the directory that you set up. Make use of escape characters and end path with a double "\\\\" e.g. "\\\\\\\example\\\root\\\directory\\\\".
 
 2) Download the epcn excel file from this [webpage](https://digital.nhs.uk/services/organisation-data-service/file-downloads/gp-and-gp-practice-related-data). Move it to the location specified in the above diagram. Copy the absolute path of this file and use it as the "epcn_path" in the config.json.
 
@@ -80,7 +80,7 @@ There are a number of acronyms used in the text. They are set out in full and ex
 
  CQRS: Calculating Quality Reporting Service. The Calculating Quality Reporting Service (CQRS) is an approvals, reporting and payments calculation system for GP practices. More information on CQRS can be found [here](https://welcome.cqrs.nhs.uk/).
 
-NCDes: Network Contract Directed Enhanced Services. This is explained in detail on this [page](https://digital.nhs.uk/data-and-information/publications/statistical/mi-network-contract-des/2022-23).
+NCDes: Network Contract Directed Enhanced Services. This is explained in detail on this [page](https://digital.nhs.uk/data-and-information/publications/statistical/mi-network-contract-des).
 
 PCN: Primary care networks. Groups of GP practices working closely together - along with other healthcare staff and organisations - providing integrated services to the local population.
 

config.toml

@@ -1,7 +1,11 @@
 #NCD publication Parameters 
+#Please check the confluence page for the standardised values
 
 [Setup]
-test_mode = "false"
+test_mode = "false"    #Must be lower case - "true" = test mode, "false" = live run
+
+[Dates]
+Publication_date = "MMM_YYYY"    #MMM_YYYY
 
 [Filepaths]
 root_directory  = "insert path to your root directory here"
@@ -22,5 +26,7 @@ removal_indicator_list = []
 bad_measure_list =  []
 
 [Pairs]
-remove_ind_pair.EHCH01 = "Numerator"
-remove_meas_pair.EHCH04 = "Denominator"
+remove_ind_pair.EHCH01 = "Denominator"
+remove_meas_pair.EHCH04 = "Numerator"
+
+

environment.yml ncdes/Archived_files/environment.ymlenvironment.yml renamed to ncdes/Archived_files/environment.yml

@@ -0,0 +1,10 @@
+@echo off
+call C:\ProgramData\Anaconda3\Scripts\activate.bat C:\ProgramData\Anaconda3
+call conda env remove --name ncdes
+call conda env create --name ncdes --file environment.yml
+call conda activate ncdes
+python -m ncdes.main
+pause
+
+
+

ncdes/Archived_files/run_ncdes.bat

@@ -1,17 +1,19 @@
 from datetime import datetime
 import pandas as pd
 import os
+from pathlib import PurePath
+import logging
 
-from ncdes.data import sql_connection
-from ncdes.data.data_load import *
-from ncdes.data import amender
+from ncdes.utils import sql_connection
+from ncdes.data_ingestion.data_load import *
+from ncdes.data_ingestion import amender
 from ncdes.processing import processing_steps
 from ncdes.processing import validation_check as check
 
-from ncdes.output import outputs
+from ncdes.data_export import outputs
 
 from ncdes.utils.adhoc_fix import remove_problem_indicators, remove_problem_measures, remove_problem_indicator_measure_pairs
-from ncdes.utils.setup import check_saved_changes
+from ncdes.utils.setup import check_saved_changes, log_setup
 import warnings
 
 warnings.simplefilter(action="ignore", category=UserWarning)
@@ -36,57 +38,69 @@ def main() -> None:
     #check that all changes made are saved and have indeed been updated ready to run - important for test runs
     check_saved_changes()
 
+    #load config parameters
     print("\n"*3,"Loading config file")
     config = get_config('config.toml')
     config_conn = config['Connections']
     config_fp = config['Filepaths']
-    
+    root_directory = config_fp["root_directory"]
     test_run = config['Setup']['test_mode']
-    print("\n"*3,f"Test mode established as: {test_run}")
+    pub_date = config['Dates']['Publication_date']
 
-    print("Establishing SQL connection")
-    connection = sql_connection.connect(server=config_conn["server"], database=config_conn["database"])
-    root_directory = config_fp["root_directory"]
 
     output_directory = outputs.test_run_change_outputs_fldr(test_run, root_directory)
-    print(f"Selecting output folder: {output_directory}")
+    log_output_directory = PurePath(output_directory, "Logs")
+    # set test or live outputs folder and create directory where this does not already exist
+    if not os.path.exists(str(log_output_directory)):
+            os.makedirs(str(log_output_directory))
+
+    # logger setup
+    log_setup(log_output_directory, pub_date)
+
+    #input logger messages around this run
+    logging.info(f"Test mode established as: {test_run} \n\n\n")
+    logging.info(f'Outputs will be written to the following folder: {output_directory}.')
+
+    logging.info("Establishing SQL connection")
+    connection = sql_connection.connect(server=config_conn["server"], database=config_conn["database"])
 
-    print("Loading NCDes data")
-    ncdes_raw = load_csvs_in_directory_as_concat_dataframe(f"{root_directory}\\Input\\Current")
+    logging.info("Loading NCDes data")
+    ncdes_raw_filepath = PurePath(root_directory, 'Input', 'Current')
+    ncdes_raw = load_csvs_in_directory_as_concat_dataframe(ncdes_raw_filepath)
     ncdes_raw_archive = ncdes_raw.copy(deep=True)
 
-    print("Amending CQRS data")
+    logging.info("Amending CQRS data")
     ncdes_raw = amender.update_dataframe(ncdes_raw, config)
 
-    print("Cleaning NCDes data")
+    logging.info("Cleaning NCDes data")
     ncdes_clean = processing_steps.clean_ncdes(ncdes_raw)
     reporting_period = processing_steps.get_formatted_reporting_end_date_from_ncdes_data(ncdes_clean)
 
     geo_ccg_sql_str, geo_reg_sql_str, stp_sql_str, prac_sql_str = get_sql_query_strings(reporting_period)
 
-    print("Loading SQL mapping data")
+    logging.info("Loading SQL mapping data")
     geo_ccg_df = pd.read_sql(sql=geo_ccg_sql_str, con=connection)
     geo_reg_df = pd.read_sql(sql=geo_reg_sql_str, con=connection)
     stp_df = pd.read_sql(sql=stp_sql_str, con=connection)
     prac_df = pd.read_sql(sql=prac_sql_str, con=connection)
 
-    print("Formatting SQL mapping data")
+    logging.info("Formatting SQL mapping data")
     geo_ccg_df, geo_reg_df, stp_df = processing_steps.sql_df_cols_to_upper_case(geo_ccg_df, geo_reg_df, stp_df)
 
-    print("Loading ePCN data")
+    logging.info("Loading ePCN data")
     raw_epcn = load_epcn_excel_table(epcn_path=config_fp["epcn_path"])
     epcn_df = processing_steps.epcn_transform(raw_epcn)
 
-    print("Creating mapping table")
+    logging.info("Creating mapping table")
     mapping_table = processing_steps.create_mapping_table(geo_ccg_df, geo_reg_df, stp_df, prac_df, epcn_df)
 
-    print("Merging NCDes data with mapping data")
+    logging.info("Merging NCDes data with mapping data")
     NCDes_with_geogs = processing_steps.merge_tables_fill_Na_reorder_cols(mapping_df=mapping_table, ncdes_df_cleaned=ncdes_clean, CORRECT_COLUMN_ORDER_NCDes_with_geogs=CORRECT_COLUMN_ORDER_NCDes_with_geogs)
 
-    print("Starting validation checks")
+    logging.info("Starting validation checks")
     check.run_all_column_has_expected_values_validations(NCDes_with_geogs, root_directory)
 
-    print("Applying suppression")
+    logging.info("Applying suppression")
     NCDes_suppressed = processing_steps.suppress_output(
         main_table=NCDes_with_geogs,
         root_directory=root_directory,
@@ -99,47 +113,47 @@ def main() -> None:
         main_table_ind_code_col_name='IND_CODE'
     )
 
-    print("Removing problem indicators:")
-    print(str(config["Indicators"]["removal_indicator_list"]))
+    #logging.info(f"Removing problem indicators: {str(config["Indicators"]["removal_indicator_list"])}")
+    logging.info("Removing problem indicators")
     NCDes_problem_ind_rem = remove_problem_indicators.remove_indicators(NCDes_suppressed,  removal_indicator_list=config["Indicators"]["removal_indicator_list"])
-    print("Removing problem measures")
+    logging.info("Removing problem measures")
     NCDes_problem_meas_rem = remove_problem_measures.remove_measures(NCDes_problem_ind_rem, bad_measure_list=config["Measures"]["bad_measure_list"])
 
-    print("Removing problem indicator-measure combinations if required")
+    logging.info("Removing problem indicator-measure combinations if required")
     NCDes_problem_meas_rem = remove_problem_indicator_measure_pairs.remove_pairs(NCDes_problem_ind_rem, bad_indicator_measure_list=config["Pairs"]['remove_ind_pair'])
     NCDes_problem_meas_rem = remove_problem_indicator_measure_pairs.remove_pairs(NCDes_problem_meas_rem, bad_indicator_measure_list=config["Pairs"]['remove_meas_pair'])
 
-    print("Joining ruleset ID to copy of output data for ruleset-specific outputs")
+    logging.info("Joining ruleset ID to copy of output data for ruleset-specific outputs")
     NCDes_with_rulesets = processing_steps.merge_data_with_ruleset_id(NCDes_problem_meas_rem, root_directory)
 
-    print("Saving main output")
+    logging.info("Saving main output")
     outputs.save_NCDes_main_to_csv(NCDes_problem_meas_rem, output_directory)
 
-    print("Zipping main output")
+    logging.info("Zipping main output")
     outputs.save_NCDes_main_to_zip(NCDes_problem_meas_rem, output_directory)
 
-    print("Saving outputs split by ruleset")
+    logging.info("Saving outputs split by ruleset")
     outputs.save_NCDes_by_ruleset_to_csvs(NCDes_with_rulesets, output_directory)
 
-    print("Zipping outputs split by ruleset")
+    logging.info("Zipping outputs split by ruleset")
     outputs.save_NCDes_by_ruleset_to_zip(NCDes_with_rulesets, output_directory)
 
-    print("Saving trend monitor")
-    if test_run == True:
-        print("Test mode: skipping trend monitor")
+    logging.info("Trend monitor updates")
+    if test_run.lower() == "true":
+        logging.info("Test mode: skipping trend monitor")
     else:
         outputs.save_trendmonitor(NCDes_problem_meas_rem, root_directory)
 
-    print("Saving excel output")
+    logging.info("Saving LDHC excel output")
     outputs.save_NCDes_main_to_excel(NCDes_problem_meas_rem, root_directory, server=config_conn["server"], database=config_conn["database"], test_run=test_run)
 
-    print("Archiving input")
-    outputs.archive_input_as_csv(ncdes_raw_archive, output_directory)
+    logging.info("Archiving input")
+    outputs.archive_input(output_directory)
 
-    print("Deleting input files from input folder")
-    outputs.remove_files_from_input_folder(path=f"{root_directory}Input\\Current\\")
+    logging.info("Deleting input files from input folder")
+    outputs.remove_files_from_input_folder(ncdes_raw_filepath)
 
-    print("Job complete")
+    logging.info("Job complete")
     outputs.open_outputs(NCDes_problem_meas_rem, output_directory)