Ubc wrf inclusion

nc2pt/climatedata.py

@@ -39,6 +39,7 @@ class ClimateModel:
     hr_ref: Optional[ClimateVariable] = None
     engine: Optional[str] = None  # Optional engine override for this model
     emulation_data: Optional[bool] = False  # Optional bool to handle metadata ingestion
+    loader: Optional[str] = "default" # Optional flag for UBC WRF specific io 'ubc_wrf'
     alignment_pipeline: List[str] = field(default_factory=lambda: [
         "temporal_crop", "regrid", "spatial_crop", "coarsen", "user_defined_transforms", "split_data"
     ])

nc2pt/conf/climate_models/hr.yaml

@@ -4,11 +4,12 @@
 
 _target_: nc2pt.climatedata.ClimateModel
 name: hr
-info: "High Resolution USask WRF, Western Canada"
+info: "High Resolution UBC WRF, Western Canada"
 alignment_pipeline: ["temporal_crop", "spatial_crop", "user_defined_transforms", "split_data"]
+loader: "ubc_wrf"
 
 climate_variables:
   - ${internal.hr_uas}
   - ${internal.hr_vas}
   - ${internal.hr_tas}
-  - ${internal.hr_pr}
+  - ${internal.hr_ps}

nc2pt/conf/climate_models/hr/ps.yaml

@@ -0,0 +1,10 @@
+
+_target_: nc2pt.climatedata.ClimateVariable
+name: "ps"
+alternative_names: ["PSFC"]
+path:  ${internal.paths.hr.ps}
+is_west_negative: true
+apply_standardize: false
+apply_normalize: true
+invariant: false
+transform: []

nc2pt/conf/climate_models/hr_invariant.yaml

@@ -9,4 +9,7 @@ engine: netcdf4
 alignment_pipeline: ['spatial_crop']
 
 climate_variables:
-  - ${internal.hr_invariant_topo}
+  - ${internal.hr_invariant_topo}
+  - ${internal.hr_invariant_land_mask}
+  - ${internal.hr_invariant_land_use}
+  - ${internal.hr_invariant_surface_roughness}

nc2pt/conf/climate_models/hr_invariant/land_mask.yaml

@@ -0,0 +1,11 @@
+# Land mask used in WRF simulations (HR Model)
+
+_target_: nc2pt.climatedata.ClimateVariable
+name: "land_mask"
+alternative_names: ["LANDMASK"]
+path: ${internal.paths.hr_invariant}
+is_west_negative: false
+invariant: true
+apply_standardize: false
+apply_normalize: true
+transform: []

nc2pt/conf/climate_models/hr_invariant/land_use.yaml

@@ -0,0 +1,11 @@
+# Topography used in WRF simulations (HR Model)
+
+_target_: nc2pt.climatedata.ClimateVariable
+name: "land_use"
+alternative_names: ["LU_INDEX"]
+path: ${internal.paths.hr_invariant}
+is_west_negative: false
+invariant: true
+apply_standardize: false
+apply_normalize: true
+transform: []

nc2pt/conf/climate_models/hr_invariant/surface_roughness.yaml

@@ -0,0 +1,11 @@
+# Topography used in WRF simulations (HR Model)
+
+_target_: nc2pt.climatedata.ClimateVariable
+name: "surface_roughness"
+alternative_names: ["VAR_SSO"]
+path: ${internal.paths.hr_invariant}
+is_west_negative: false
+invariant: true
+apply_standardize: false
+apply_normalize: true
+transform: []

nc2pt/conf/climate_models/lr.yaml

@@ -12,14 +12,14 @@ alignment_pipeline: ["temporal_crop", "regrid", "spatial_crop", "coarsen", "user
 hr_ref:
   _target_: nc2pt.climatedata.ClimateVariable
   name: "hr_ref"
-  alternative_names: ["T2"]
+  alternative_names: ["HGT"]
   path: ${internal.paths.hr_ref}
   is_west_negative: true
   invariant: true
 
 climate_variables:
   # Activate variables to include in the LR model
   - ${internal.lr_uas}
-  #- ${internal.lr_vas}
-  #- ${internal.lr_tas}
-  #- ${internal.lr_pr}
+  - ${internal.lr_vas}
+  - ${internal.lr_tas}
+  - ${internal.lr_ps}

nc2pt/conf/climate_models/lr/ps.yaml

@@ -0,0 +1,9 @@
+_target_: nc2pt.climatedata.ClimateVariable
+name: "ps"
+alternative_names: ["PSFC"]
+path:  ${internal.paths.lr.ps}
+is_west_negative: false
+apply_standardize: false
+apply_normalize: true
+invariant: false
+transform: []

nc2pt/conf/climate_models/lr/tas.yaml

@@ -4,7 +4,7 @@ _target_: nc2pt.climatedata.ClimateVariable
 name: "tas"
 alternative_names: ["T2", "surface temperature"]
 path: ${internal.paths.lr.tas}
-is_west_negative: true
+is_west_negative: false
 apply_standardize: false
 apply_normalize: true
 invariant: false

nc2pt/conf/climate_models/lr/uas.yaml

@@ -5,8 +5,8 @@ name: "uas"
 alternative_names: ["U10", "u10", "uas"]
 path: ${internal.paths.lr.uas}
 is_west_negative: false
-apply_standardize: true
-apply_normalize: false
+apply_standardize: false
+apply_normalize: true
 invariant: false
 coarsening_method: "mean"
 transform: []

nc2pt/conf/climate_models/lr_invariant.yaml

@@ -9,4 +9,7 @@ engine: netcdf4
 alignment_pipeline: ['spatial_crop', 'coarsen']
 
 climate_variables:
-  - ${internal.hr_invariant_topo}
+  - ${internal.hr_invariant_topo}
+  - ${internal.hr_invariant_land_mask}
+  - ${internal.hr_invariant_land_use}
+  - ${internal.hr_invariant_surface_roughness}

nc2pt/conf/coords.yaml

@@ -5,10 +5,10 @@
 coords:
   - _target_: nc2pt.climatedata.ClimateDimension
     name: lat
-    alternative_names: ["latitude", "Lat", "Latitude"]
+    alternative_names: ["latitude", "Lat", "Latitude","XLAT"]
     chunksize: 100
 
   - _target_: nc2pt.climatedata.ClimateDimension
     name: lon
-    alternative_names: ["longitude", "Long", "Lon", "Longitude"]
+    alternative_names: ["longitude", "Long", "Lon", "Longitude", "XLONG"]
     chunksize: 100

nc2pt/conf/injections.yaml

@@ -26,6 +26,9 @@ defaults:
   # HR Invariant Model
   - climate_models/hr_invariant@internal._hr_invariant_model
   - climate_models/hr_invariant/topo@internal._hr_invariant_topo
+  - climate_models/hr_invariant/land_mask@internal._hr_invariant_land_mask
+  - climate_models/hr_invariant/land_use@internal._hr_invariant_land_use
+  - climate_models/hr_invariant/surface_roughness@internal._hr_invariant_surface_roughness
 
   # LR_emulation Model
   - climate_models/lr_emulation@internal._lr_emulation_model
@@ -63,6 +66,9 @@ internal:
   # HR Invariant aliases
   hr_invariant: ${internal._hr_invariant_model}
   hr_invariant_topo: ${internal._hr_invariant_topo}
+  hr_invariant_land_mask: ${internal._hr_invariant_land_mask}
+  hr_invariant_land_use: ${internal._hr_invariant_land_use}
+  hr_invariant_surface_roughness: ${internal._hr_invariant_surface_roughness}
 
   # LR Emulation aliases
   lr_emulation: ${internal._lr_emulation_model}

nc2pt/conf/paths.yaml

@@ -7,16 +7,16 @@ paths:
   # ────────────────────────────────────────
   # User-dependent path for HR reference grid
   # ────────────────────────────────────────
-  hr_ref: /home/username/projects/nc2pt/nc2pt/data/hr_ref.nc
+  hr_ref: /net/venus/kenes/data/sbeairsto/UBC_WRF/ubc_wrf_staticfields_d03.nc # UBC_WRF reference grid (should be accessible to everyone)
 
   # ────────────────────────────────────────
   # Shared data paths – High-Resolution WRF (HR)
   # ────────────────────────────────────────
   hr:
-    uas: /net/venus/kenes/downloaded-data/acannon/USask-WRF-WCA/ctl-wrf-wca/U10/*.nc
-    vas: /net/venus/kenes/downloaded-data/acannon/USask-WRF-WCA/ctl-wrf-wca/V10/*.nc
-    pr:  /net/venus/kenes/downloaded-data/acannon/USask-WRF-WCA/ctl-wrf-wca/PREC/*.nc
-    tas: /net/venus/kenes/downloaded-data/acannon/USask-WRF-WCA/ctl-wrf-wca/T2/*.nc
+    uas: /net/venus/blue/data/WRF/kikridaust/Share_Data/SUBSETTED/*/metgrid_*/COMPRESSED_SUBSETTED_d03_metgrid_*.nc
+    vas: /net/venus/blue/data/WRF/kikridaust/Share_Data/SUBSETTED/*/metgrid_*/COMPRESSED_SUBSETTED_d03_metgrid_*.nc
+    pr:  /net/venus/blue/data/WRF/kikridaust/Share_Data/SUBSETTED/*/metgrid_*/COMPRESSED_SUBSETTED_d03_metgrid_*.nc
+    tas: /net/venus/blue/data/WRF/kikridaust/Share_Data/SUBSETTED/*/metgrid_*/COMPRESSED_SUBSETTED_d03_metgrid_*.nc
 
   # ────────────────────────────────────────
   # Shared data paths – Low-Resolution ERA5 (LR)
@@ -30,16 +30,16 @@ paths:
   # ────────────────────────────────────────
   # Shared invariant fields (e.g., orography, landmask)
   # ────────────────────────────────────────
-  hr_invariant: /net/venus/kenes/downloaded-data/acannon/USask-WRF-WCA/CA4km_const.nc
+  hr_invariant: /net/venus/kenes/data/sbeairsto/UBC_WRF/ubc_wrf_staticfields_d03.nc # UBC_WRF static fields (should be accessible to everyone)
 
   # ────────────────────────────────────────
   # User-dependent path for – Low-Resolution ERA5 Emulation Metadata (LR Emulation) 
   # ────────────────────────────────────────
   lr_emulation:
-    uas_metadata: /home/username/data/path/lr_uas_feature_scaling_metadata.json
-    vas_metadata: /home/username/data/path/lr_vas_feature_scaling_metadata.json
-    pr_metadata:  /home/username/data/path/lr_pr_feature_scaling_metadata.json
-    tas_metadata: /home/username/data/path/lr_tas_feature_scaling_metadata.json
+      uas_metadata: /home/username/data/path/lr_uas_feature_scaling_metadata.json
+      vas_metadata: /home/username/data/path/lr_vas_feature_scaling_metadata.json
+      pr_metadata:  /home/username/data/path/lr_pr_feature_scaling_metadata.json
+      tas_metadata: /home/username/data/path/lr_tas_feature_scaling_metadata.json
 
   # ────────────────────────────────────────
   # User-dependent path for My Model Template

nc2pt/conf/select.yaml

@@ -7,20 +7,20 @@ select:
   time:
     # Define the full available date range
     range:
-      start: "20001201T06:00:00"
-      end: "20010101T12:00:00"
+      start: "20140101T00:00:00"
+      end: "20170101T00:00:00"
 
     # Select years to reserve for testing and validation
     # Remaining years are used for training
-    test_years: [2001]
-    validation_years: [None]
+    test_years: [2014]
+    validation_years: [2015]
 
   # ───── Spatial subsetting ─────
   spatial:
     scale_factor: 8  # Factor to downscale HR data to LR
     x:
-      first_index: 110
-      last_index: 622
+      first_index: 125
+      last_index: 253
     y:
-      first_index: 20
-      last_index: 532
+      first_index: 100
+      last_index: 228

nc2pt/io.py

@@ -1,21 +1,34 @@
 from datetime import datetime
 import xarray as xr
+import pandas as pd
+import re
+from pathlib import Path
 
 
-def load_grid(path: str, engine: str = "netcdf4", chunks: int = 250) -> xr.Dataset:
+def load_grid(path: str, engine: str = "netcdf4", loader: str = "default", chunks: int = 250) -> xr.Dataset:
     """Load the grid to regrid to.
 
     Parameters
     ----------
     path : str
         Path to the grid to regrid to.
+    engine : str
+        Engine to use for loading.
+    chunks : int
+        Chunk size for dask.
+    loader : str
+        Loader type: 'default', 'ubc_wrf'
 
     Returns
     -------
     grid : xarray.Dataset
         Grid to regrid to.
     """
+    if loader == "ubc_wrf":
+        from ubc_wrf_io import load_ubc_wrf
+        return load_ubc_wrf(path, engine=engine, chunks=chunks)
 
+    # Default loader
     if "*" in path or isinstance(path, list):
         with xr.open_mfdataset(path, engine=engine, parallel=True, chunks="auto") as ds:
             return ds
@@ -41,4 +54,4 @@ def write_to_zarr(ds: xr.Dataset, path: str) -> None:
             "history": f"Created by {__file__} on {datetime.now()}",
         }
     )
-    ds.to_zarr(f"{path}.zarr", mode="w", consolidated=True)
+    ds.to_zarr(f"{path}.zarr", mode="w", consolidated=True)

nc2pt/preprocess.py

@@ -40,8 +40,9 @@ def preprocess_variables(model: ClimateModel, climdata: ClimateData) -> None:
         # Instantiates climate_variable object in cliamtedata.py
         climate_variable = instantiate(climate_variable)
         engine = model.engine or climdata.compute.engine
+        loader = model.loader
 
-        ds = load_grid(climate_variable.path, engine=engine)
+        ds = load_grid(climate_variable.path, engine=engine, loader=loader)
 
         start = timer()
         logging.info(

nc2pt/ubc_wrf_io.py

@@ -0,0 +1,79 @@
+"""UBC WRF-specific data loading utilities."""
+
+import xarray as xr
+import pandas as pd
+import re
+from pathlib import Path
+
+
+def get_ubc_wrf_file_list(pattern: str) -> list:
+    """Get file list"""
+    import glob
+
+    print("Scanning for files (this may take ~30s over NFS)...")
+    files = sorted(glob.glob(pattern, recursive=True))
+    print(f"Found {len(files)} files")
+
+    return files
+
+
+def add_ubc_wrf_timesteps(ds):
+    """
+    Preprocess WRF metgrid files with dummy Time coordinate.
+    Extracts date from filename and creates proper time axis.
+    Drops last timestep (corresponding to next month's 00:00:00).
+    """
+    # Get filename from dataset encoding
+    filepath = ds.encoding.get('source', '')
+
+    # Extract year and month: metgrid_YYYY_MM.nc
+    match = re.search(r'metgrid_(\d{4})_(\d{2})\.nc$', filepath)
+
+    if match:
+        year, month = match.groups()
+        start_date = f"{year}-{month}-01"
+
+        # Drop last timestep (spin-up for next month)
+        ds = ds.isel(Times=slice(None, -1))
+
+        # Create time coordinates for remaining timesteps
+        n_times = ds.sizes['Times']
+        time_coords = pd.date_range(start=start_date, periods=n_times, freq='h')
+        ds = ds.assign_coords(Times=time_coords)
+    else:
+        # For invariant fields or unparseable files, drop Time if it's dummy
+        if 'Times' in ds.dims and len(ds.Times) <= 2:
+            ds = ds.isel(Times=0, drop=True)
+
+    return ds
+
+
+def load_ubc_wrf(path: str, engine: str = "netcdf4", chunks: str = "auto") -> xr.Dataset:
+    """Load WRF metgrid files with proper time coordinate handling."""
+
+    if "*" in path or isinstance(path, list):
+        if isinstance(path, str):
+            file_list = get_ubc_wrf_file_list(path)
+        else:
+            file_list = path
+
+        print(f"Opening {len(file_list)} files...")
+        print("Note: Dropping last timestep of each month (corresponds to M+1 00:00:00)")  # Log once here
+
+        ds = xr.open_mfdataset(
+            file_list,
+            engine=engine,
+            parallel=True,
+            chunks='auto',
+            preprocess=add_ubc_wrf_timesteps,
+            combine='nested',
+            concat_dim='Times',
+            combine_attrs='override',
+            data_vars='minimal',
+            coords='minimal',
+            compat='override'
+        )
+        return ds
+    else:
+        ds = xr.open_dataset(path, engine=engine, chunks=chunks)
+        return add_ubc_wrf_timesteps(ds)