From a378266f3fd301d504299a65ba0752b1fb9520f4 Mon Sep 17 00:00:00 2001 From: Nabil Freij Date: Mon, 16 Dec 2024 17:18:39 -0800 Subject: [PATCH 01/11] Functions now require tables to be passed in --- aiapy/calibrate/meta.py | 30 +-- aiapy/calibrate/prep.py | 59 ++--- aiapy/calibrate/spikes.py | 11 +- aiapy/calibrate/transform.py | 4 + aiapy/calibrate/util.py | 249 +++++++++++++----- aiapy/conftest.py | 17 +- aiapy/util/__init__.py | 2 + aiapy/util/net.py | 49 ++++ aiapy/util/tests/test_net.py | 14 + aiapy/util/util.py | 9 +- ...adation.py => skip_correct_degradation.py} | 0 ...data.py => skip_download_specific_data.py} | 0 ...t_pixels.py => skip_replace_hot_pixels.py} | 0 ruff.toml | 5 + 14 files changed, 297 insertions(+), 152 deletions(-) create mode 100644 aiapy/util/net.py create mode 100644 aiapy/util/tests/test_net.py rename examples/{correct_degradation.py => skip_correct_degradation.py} (100%) rename examples/{download_specific_data.py => skip_download_specific_data.py} (100%) rename examples/{replace_hot_pixels.py => skip_replace_hot_pixels.py} (100%) diff --git a/aiapy/calibrate/meta.py b/aiapy/calibrate/meta.py index 328e61b..7371873 100644 --- a/aiapy/calibrate/meta.py +++ b/aiapy/calibrate/meta.py @@ -12,7 +12,6 @@ from sunpy.map import contains_full_disk -from aiapy.calibrate.util import get_pointing_table from aiapy.util.exceptions import AIApyUserWarning __all__ = ["fix_observer_location", "update_pointing"] @@ -42,7 +41,7 @@ def fix_observer_location(smap): ------- `~sunpy.map.sources.AIAMap` """ - # Create observer coordinate from HAE coordinates + # Create observer coordinate from HAE coordinates (reason?) coord = SkyCoord( x=smap.meta["haex_obs"] * u.m, y=smap.meta["haey_obs"] * u.m, @@ -51,37 +50,26 @@ def fix_observer_location(smap): frame=HeliocentricMeanEcliptic, obstime=smap.reference_date, ).heliographic_stonyhurst - # Update header new_meta = copy.deepcopy(smap.meta) new_meta["hgln_obs"] = coord.lon.to(u.degree).value new_meta["hglt_obs"] = coord.lat.to(u.degree).value new_meta["dsun_obs"] = coord.radius.to(u.m).value - return smap._new_instance(smap.data, new_meta, plot_settings=smap.plot_settings, mask=smap.mask) -def update_pointing(smap, *, pointing_table=None): +def update_pointing(smap, *, pointing_table): """ Update the pointing information in the input map header. This function updates the pointing information in ``smap`` by updating the ``CRPIX1, CRPIX2, CDELT1, CDELT2, CROTA2`` keywords in the header using the information provided in ``pointing_table``. - If ``pointing_table`` is not specified, the 3-hour pointing - information is queried from the `JSOC `_. .. note:: The method removes any ``PCi_j`` matrix keys in the header and updates the ``CROTA2`` keyword. - .. note:: - - If correcting pointing information for a large number of images, - it is strongly recommended to query the table once for the - appropriate interval and then pass this table in rather than - executing repeated queries. - .. warning:: This function is only intended to be used for full-disk images @@ -92,13 +80,14 @@ def update_pointing(smap, *, pointing_table=None): ---------- smap : `~sunpy.map.sources.AIAMap` Input map. - pointing_table : `~astropy.table.QTable`, optional - Table of pointing information. If not specified, the table - will be retrieved from JSOC. + pointing_table : `~astropy.table.QTable` + Table of pointing information. + You can get this table by calling `aiapy.calibrate.util.get_pointing_table`. Returns ------- `~sunpy.map.sources.AIAMap` + Updated map with pointing information. See Also -------- @@ -111,9 +100,6 @@ def update_pointing(smap, *, pointing_table=None): if not all(d == (s * u.pixel) for d, s in zip(smap.dimensions, shape_full_frame, strict=True)): msg = f"Input must be at the full resolution of {shape_full_frame}" raise ValueError(msg) - if pointing_table is None: - # Make range wide enough to get closest 3-hour pointing - pointing_table = get_pointing_table(smap.date - 12 * u.h, smap.date + 12 * u.h) # Find row in which T_START <= T_OBS < T_STOP # The following notes are from a private communication with J. Serafin (LMSAL) # and are preserved here to explain the reasoning for selecting the particular @@ -176,9 +162,7 @@ def update_pointing(smap, *, pointing_table=None): ) else: new_meta[key] = value - - # sunpy map converts crota to a PCi_j matrix, so we remove it to force the - # re-conversion. + # sunpy.map.Map converts crota to a PCi_j matrix, so we remove it to force the re-conversion. new_meta.pop("PC1_1") new_meta.pop("PC1_2") new_meta.pop("PC2_1") diff --git a/aiapy/calibrate/prep.py b/aiapy/calibrate/prep.py index 483361f..f50414d 100644 --- a/aiapy/calibrate/prep.py +++ b/aiapy/calibrate/prep.py @@ -13,7 +13,7 @@ from sunpy.util.decorators import add_common_docstring from aiapy.calibrate.transform import _rotation_function_names -from aiapy.calibrate.util import _select_epoch_from_correction_table, get_correction_table +from aiapy.calibrate.util import _select_epoch_from_correction_table from aiapy.util import AIApyUserWarning from aiapy.util.decorators import validate_channel @@ -101,7 +101,7 @@ def register(smap, *, missing=None, order=3, method="scipy"): missing=missing, method=method, ) - # extract center from padded smap.rotate output + # Extract center from padded smap.rotate output # crpix1 and crpix2 will be equal (recenter=True), as prep does not work with submaps center = np.floor(tempmap.meta["crpix1"]) range_side = (center + np.array([-1, 1]) * smap.data.shape[0] / 2) * u.pix @@ -115,7 +115,7 @@ def register(smap, *, missing=None, order=3, method="scipy"): return newmap -def correct_degradation(smap, *, correction_table=None, calibration_version=None): +def correct_degradation(smap, *, correction_table): """ Apply time-dependent degradation correction to an AIA map. @@ -128,22 +128,14 @@ def correct_degradation(smap, *, correction_table=None, calibration_version=None ---------- smap : `~sunpy.map.sources.AIAMap` Map to be corrected. - correction_table : `~astropy.table.Table` or `str`, optional - Table of correction parameters or path to correction table file. - If not specified, it will be queried from JSOC. See - `aiapy.calibrate.util.get_correction_table` for more information. - If you are processing many images, it is recommended to - read the correction table once and pass it with this argument to avoid - multiple redundant network calls. - calibration_version : `int`, optional - The version of the calibration to use when calculating the degradation. - By default, this is the most recent version available from JSOC. If you - are using a specific calibration response file, you may need to specify - this according to the version in that file. + correction_table : `~astropy.table.Table` + Table of correction parameters. + You can get this table by calling `aiapy.calibrate.util.get_correction_table`. Returns ------- `~sunpy.map.sources.AIAMap` + Degradation-corrected map. See Also -------- @@ -153,7 +145,6 @@ def correct_degradation(smap, *, correction_table=None, calibration_version=None smap.wavelength, smap.date, correction_table=correction_table, - calibration_version=calibration_version, ) return smap / d @@ -164,8 +155,7 @@ def degradation( channel: u.angstrom, obstime, *, - correction_table=None, - calibration_version=None, + correction_table, ) -> u.dimensionless_unscaled: r""" Correction to account for time-dependent degradation of the instrument. @@ -195,19 +185,17 @@ def degradation( Parameters ---------- channel : `~astropy.units.Quantity` + Wavelength of the channel. obstime : `~astropy.time.Time` - correction_table : `~astropy.table.Table` or `str`, optional - Table of correction parameters or path to correction table file. - If not specified, it will be queried from JSOC. See - `aiapy.calibrate.util.get_correction_table` for more information. - If you are processing many images, it is recommended to - read the correction table once and pass it with this argument to avoid - multiple redundant network calls. - calibration_version : `int`, optional - The version of the calibration to use when calculating the degradation. - By default, this is the most recent version available from JSOC. If you - are using a specific calibration response file, you may need to specify - this according to the version in that file. + Observation time. + correction_table : `~astropy.table.Table` + Table of correction parameters. + You can get this table by calling `aiapy.calibrate.util.get_correction_table`. + + Returns + ------- + `~astropy.units.Quantity` + Degradation correction factor. See Also -------- @@ -219,15 +207,12 @@ def degradation( obstime = obstime.reshape((1,)) ratio = np.zeros(obstime.shape) poly = np.zeros(obstime.shape) - # Do this outside of the loop to avoid repeated queries - correction_table = get_correction_table(correction_table=correction_table) - for i, t in enumerate(obstime): - table = _select_epoch_from_correction_table(channel, t, correction_table, version=calibration_version) - + for idx, t in enumerate(obstime): + table = _select_epoch_from_correction_table(channel, t, correction_table) # Time difference between obstime and start of epoch dt = (t - table["T_START"][-1]).to(u.day).value # Correction to most recent epoch - ratio[i] = table["EFF_AREA"][-1] / table["EFF_AREA"][0] + ratio[idx] = table["EFF_AREA"][-1] / table["EFF_AREA"][0] # Polynomial correction to interpolate within epoch - poly[i] = table["EFFA_P1"][-1] * dt + table["EFFA_P2"][-1] * dt**2 + table["EFFA_P3"][-1] * dt**3 + 1.0 + poly[idx] = table["EFFA_P1"][-1] * dt + table["EFFA_P2"][-1] * dt**2 + table["EFFA_P3"][-1] * dt**3 + 1.0 return u.Quantity(poly * ratio) diff --git a/aiapy/calibrate/spikes.py b/aiapy/calibrate/spikes.py index 83299d2..c5ba07b 100644 --- a/aiapy/calibrate/spikes.py +++ b/aiapy/calibrate/spikes.py @@ -11,11 +11,11 @@ from astropy.io import fits from astropy.wcs.utils import pixel_to_pixel -import drms from sunpy.map.mapbase import PixelPair from sunpy.map.sources.sdo import AIAMap from aiapy.util import AIApyUserWarning +from aiapy.util.net import get_data_from_jsoc __all__ = ["fetch_spikes", "respike"] @@ -145,13 +145,8 @@ def fetch_spikes(smap, *, as_coords=False): array-like Original intensity values of the spikes """ - series = r"aia.lev1_euv_12s" - if smap.wavelength in (1600, 1700, 4500) * u.angstrom: - series = r"aia.lev1_uv_24s" - file = drms.Client().query( - f'{series}[{smap.date}/12s][WAVELNTH={smap.meta["wavelnth"]}]', - seg="spikes", - ) + series = "aia.lev1_uv_24s" if smap.wavelength in (1600, 1700, 4500) * u.angstrom else "aia.lev1_euv_12s" + file = get_data_from_jsoc(f'{series}[{smap.date}/12s][WAVELNTH={smap.meta["wavelnth"]}]', seg="spikes") _, spikes = fits.open(f'http://jsoc.stanford.edu{file["spikes"][0]}') # Loaded as floats, but they are actually integers spikes = spikes.data.astype(np.int32) diff --git a/aiapy/calibrate/transform.py b/aiapy/calibrate/transform.py index d14aafe..b65d496 100644 --- a/aiapy/calibrate/transform.py +++ b/aiapy/calibrate/transform.py @@ -1,3 +1,7 @@ +""" +AIA specific image transformations. +""" + from sunpy.image.transform import _rotation_registry, add_rotation_function __all__ = ["_rotation_cupy", "_rotation_function_names"] diff --git a/aiapy/calibrate/util.py b/aiapy/calibrate/util.py index bfb1438..ecceb23 100644 --- a/aiapy/calibrate/util.py +++ b/aiapy/calibrate/util.py @@ -2,7 +2,6 @@ Utilities for computing intensity corrections. """ -import os import pathlib import warnings from urllib.parse import urljoin @@ -12,18 +11,27 @@ import astropy.io.ascii import astropy.units as u +from astropy.io import ascii as astropy_ascii from astropy.table import QTable from astropy.time import Time -import drms from sunpy import log -from sunpy.net import attrs, jsoc from aiapy import _SSW_MIRRORS from aiapy.data._manager import manager from aiapy.util.decorators import validate_channel +from aiapy.util.net import get_data_from_jsoc -__all__ = ["get_correction_table", "get_error_table", "get_pointing_table"] +__all__ = [ + "CALIBRATION_VERSION", + "ERROR_VERSION", + "URL_HASH_ERROR_TABLE", + "URL_HASH_POINTING_TABLE", + "URL_HASH_RESPONSE_TABLE", + "get_correction_table", + "get_error_table", + "get_pointing_table", +] # Default version of the degradation calibration curve to use. # This needs to be incremented as the calibration is updated in JSOC. @@ -33,7 +41,7 @@ # Most recent version number for error tables; increment as new versions become available ERROR_VERSION = 3 # URLs and SHA-256 hashes for each version of the error tables -URL_HASH = { +URL_HASH_ERROR_TABLE = { 2: ( [urljoin(mirror, AIA_ERROR_FILE.format(2)) for mirror in _SSW_MIRRORS], "ac97ccc48057809723c27e3ef290c7d78ee35791d9054b2188baecfb5c290d0a", @@ -43,9 +51,89 @@ "66ff034923bb0fd1ad20e8f30c7d909e1a80745063957dd6010f81331acaf894", ), } +URL_HASH_POINTING_TABLE = ( + "https://aia.lmsal.com/public/master_aia_pointing3h.csv", + "a2c80fa0ea3453c62c91f51df045ae04b771d5cbb51c6495ed56de0da2a5482e", +) +URL_HASH_RESPONSE_TABLE = { + 10: ( + [urljoin(mirror, "sdo/aia/response/aia_V10_20201119_190000_response_table.txt") for mirror in _SSW_MIRRORS], + "0a3f2db39d05c44185f6fdeec928089fb55d1ce1e0a805145050c6356cbc6e98", + ), + 9: ( + [urljoin(mirror, "sdo/aia/response/aia_V9_20200706_215452_response_table.txt") for mirror in _SSW_MIRRORS], + "f24b384cba9935ae2e8fd3c0644312720cb6add95c49ba46f1961ae4cf0865f9", + ), + 8: ( + [urljoin(mirror, "sdo/aia/response/aia_V8_20171210_050627_response_table.txt") for mirror in _SSW_MIRRORS], + "0e8bc6af5a69f80ca9d4fc2a27854681b76574d59eb81d7201b7f618081f0fdd", + ), + 7: ( + [urljoin(mirror, "sdo/aia/response/aia_V7_20171129_195626_response_table.txt") for mirror in _SSW_MIRRORS], + "ac2171d549bd6cc6c37e13e505eef1bf0c89fc49bffd037e4ac64f0b895063ac", + ), + 6: ( + [urljoin(mirror, "sdo/aia/response/aia_V6_20141027_230030_response_table.txt") for mirror in _SSW_MIRRORS], + "11c148f447d4538db8fd247f74c26b4ae673355e2536f63eb48f9a267e58c7c6", + ), + 4: ( + [urljoin(mirror, "sdo/aia/response/aia_V4_20130109_204835_response_table.txt") for mirror in _SSW_MIRRORS], + "7e73f4effa9a8dc55f7b4993a8d181419ef555bf295c4704703ca84d7a0fc3c1", + ), + 3: ( + [urljoin(mirror, "sdo/aia/response/aia_V3_20120926_201221_response_table.txt") for mirror in _SSW_MIRRORS], + "0a5d2c2ed1cda18bb9fbdbd51fbf3374e042d20145150632ac95350fc99de68b", + ), + 2: ( + [urljoin(mirror, "sdo/aia/response/aia_V2_20111129_000000_response_table.txt") for mirror in _SSW_MIRRORS], + "d55ccd6cb3cb4bd1c688f8663f942f8a872c918a2504e5e474aa97dff45b62c9", + ), +} + +def _fetch_response_table(version: int): + # Until the delayed feature from sunpy (v6.1) is out, this function + # will need to be like this. + if version not in URL_HASH_RESPONSE_TABLE: + msg = f"Invalid response table version: {version}" + raise ValueError(msg) -def get_correction_table(*, correction_table=None): + @manager.require("response_table_v10", *URL_HASH_RESPONSE_TABLE[10]) + def fetch_response_table_v10(): + return manager.get("response_table_v10") + + @manager.require("response_table_v9", *URL_HASH_RESPONSE_TABLE[9]) + def fetch_response_table_v9(): + return manager.get("response_table_v9") + + @manager.require("response_table_v8", *URL_HASH_RESPONSE_TABLE[8]) + def fetch_response_table_v8(): + return manager.get("response_table_v8") + + @manager.require("response_table_v7", *URL_HASH_RESPONSE_TABLE[7]) + def fetch_response_table_v7(): + return manager.get("response_table_v7") + + @manager.require("response_table_v6", *URL_HASH_RESPONSE_TABLE[6]) + def fetch_response_table_v6(): + return manager.get("response_table_v6") + + @manager.require("response_table_v4", *URL_HASH_RESPONSE_TABLE[4]) + def fetch_response_table_v4(): + return manager.get("response_table_v4") + + @manager.require("response_table_v3", *URL_HASH_RESPONSE_TABLE[3]) + def fetch_response_table_v3(): + return manager.get("response_table_v3") + + @manager.require("response_table_v2", *URL_HASH_RESPONSE_TABLE[2]) + def fetch_response_table_v2(): + return manager.get("response_table_v2") + + return locals()[f"fetch_response_table_v{version}"]() + + +def get_correction_table(*, source): """ Return table of degradation correction factors. @@ -59,34 +147,35 @@ def get_correction_table(*, correction_table=None): Parameters ---------- - correction_table: `str` or `~astropy.table.QTable`, optional - Path to correction table file or an existing correction table. If None, - the table will be queried from JSOC. + source: pathlib.Path, str or int + The source of the correction table. If it is a `pathlib.Path`, it must be a file. + A string file path will error as an invalid source. + If a string, it must be "jsoc" which will fetch the most recent version from the JSOC, + otherwise an integer: 3, 4, 6, 7, 8, 9, 10 to use fixed version files from SSW. Returns ------- `~astropy.table.QTable` + Table of degradation correction factors. See Also -------- aiapy.calibrate.degradation """ - if isinstance(correction_table, astropy.table.QTable): - return correction_table - if correction_table is not None: - if isinstance(correction_table, str | pathlib.Path): - table = QTable(astropy.io.ascii.read(correction_table)) - else: - msg = "correction_table must be a file path, an existing table, or None." - raise ValueError(msg) - else: + if isinstance(source, pathlib.Path): + table = QTable(astropy.io.ascii.read(source)) + elif source in URL_HASH_RESPONSE_TABLE: + table = QTable(astropy.io.ascii.read(_fetch_response_table(source))) + elif source.lower() == "jsoc": # NOTE: the [!1=1!] disables the drms PrimeKey logic and enables # the query to find records that are ordinarily considered # identical because the PrimeKeys for this series are WAVE_STR # and T_START. Without the !1=1! the query only returns the # latest record for each unique combination of those keywords. - table = drms.Client().query("aia.response[][!1=1!]", key="**ALL**") - table = QTable.from_pandas(table) + table = QTable.from_pandas(get_data_from_jsoc(query="aia.response[][!1=1!]", key="**ALL**")) + else: + msg = "correction_table must be a file path (pathlib.Path), 'jsoc' or one of 3, 4, 6, 7, 8, 9, 10. Not {source}" + raise ValueError(msg) selected_cols = [ "DATE", "VER_NUM", @@ -117,7 +206,7 @@ def get_correction_table(*, correction_table=None): @u.quantity_input @validate_channel("channel") -def _select_epoch_from_correction_table(channel: u.angstrom, obstime, table, *, version=None): +def _select_epoch_from_correction_table(channel: u.angstrom, obstime, table): """ Return correction table with only the first epoch and the epoch in which ``obstime`` falls and for only one given calibration version. @@ -127,21 +216,18 @@ def _select_epoch_from_correction_table(channel: u.angstrom, obstime, table, *, channel : `~astropy.units.Quantity` obstime : `~astropy.time.Time` table : `~astropy.table.QTable` - version : `int` """ - version = CALIBRATION_VERSION if version is None else version # Select only this channel # NOTE: The WAVE_STR prime keys for the aia.response JSOC series for the # non-EUV channels do not have a thick/thin designation thin = "_THIN" if channel not in (1600, 1700, 4500) * u.angstrom else "" wave = channel.to(u.angstrom).value table = table[table["WAVE_STR"] == f"{wave:.0f}{thin}"] - table = table[table["VER_NUM"] == version] table.sort("DATE") # Newest entries will be last if len(table) == 0: extra_msg = " Max version is 3." if channel == 4500 * u.AA else "" raise ValueError( - f"Correction table does not contain calibration for version {version} for {channel}." + extra_msg, + f"Correction table does not contain calibration for {channel}." + extra_msg, ) # Select the epoch for the given observation time obstime_in_epoch = np.logical_and(obstime >= table["T_START"], obstime < table["T_STOP"]) @@ -159,9 +245,16 @@ def _select_epoch_from_correction_table(channel: u.angstrom, obstime, table, *, return QTable(table[[0, i_epoch[-1]]]) -def get_pointing_table(start, end): +@manager.require("pointing_table", *URL_HASH_POINTING_TABLE) +def fetch_pointing_table(): + manager.get("pointing_table") + + +def get_pointing_table(start, end, *, source): """ - Retrieve 3-hourly master pointing table from the JSOC. + Retrieve 3-hourly master pointing table from the given source. + + This function can either fetch the pointing table from JSOC or from aia.lmsal.com. This function queries `JSOC `__ for the 3-hourly master pointing table (MPT) in the interval defined by @@ -184,7 +277,13 @@ def get_pointing_table(start, end): Parameters ---------- start : `~astropy.time.Time` + Start time of the interval. end : `~astropy.time.Time` + End time of the interval. + source : str + Name of the source from which to retrieve the pointing table. + Must be one of ``"jsoc"`` or ``"lmsal"``. + Note that the LMSAL pointing table is not updated frequently. Returns ------- @@ -194,17 +293,13 @@ def get_pointing_table(start, end): -------- aiapy.calibrate.update_pointing """ - q = jsoc.JSOCClient().search( - attrs.Time(start, end=end), - attrs.jsoc.Series.aia_master_pointing3h, - ) - table = QTable(q) - if len(table.columns) == 0: - # If there's no pointing information available between these times, - # JSOC will raise a cryptic KeyError - # (see https://github.com/LM-SAL/aiapy/issues/71) - msg = f"Could not find any pointing information between {start} and {end}" - raise RuntimeError(msg) + if source.lower() == "jsoc": + table = get_data_from_jsoc(query=f"aia.master_pointing3h[{start.isot}Z-{end.isot}Z]", key="**ALL**") + elif source.lower() == "lmsal": + table = QTable(astropy_ascii.read(fetch_pointing_table())) + else: + msg = f"Invalid source: {source}, must be one of 'jsoc' or 'lmsal'" + raise ValueError(msg) table["T_START"] = Time(table["T_START"], scale="utc") table["T_STOP"] = Time(table["T_STOP"], scale="utc") for c in table.colnames: @@ -214,42 +309,70 @@ def get_pointing_table(start, end): table[c].unit = "arcsecond / pixel" if "INSTROT" in c: table[c].unit = "degree" - # Remove masking on columns with pointing parameters for c in table.colnames: + # Remove masking on columns with pointing parameters if any(n in c for n in ["X0", "Y0", "IMSCALE", "INSTROT"]) and hasattr(table[c], "mask"): table[c] = table[c].filled(np.nan) return table -def get_error_table(error_table=None): - if error_table is None: - # This is to work around a parfive bug - # https://github.com/Cadair/parfive/issues/121 - os.environ["PARFIVE_DISABLE_RANGE"] = "1" - error_table = fetch_error_table() - os.environ.pop("PARFIVE_DISABLE_RANGE") - if isinstance(error_table, str | pathlib.Path): - table = astropy.io.ascii.read(error_table) - elif isinstance(error_table, QTable): - table = error_table +def _fetch_error_table(version: int): + # Until the delayed feature from sunpy (v6.1) is out, this function + # will need to be like this. + @manager.require("error_table_v2", *URL_HASH_ERROR_TABLE[2]) + def fetch_error_table_v2(): + return manager.get("error_table_v2") + + @manager.require("error_table_v3", *URL_HASH_ERROR_TABLE[3]) + def fetch_error_table_v3(): + return manager.get("error_table_v3") + + if version == 2: + return fetch_error_table_v2() + if version == 3: + return fetch_error_table_v3() + msg = f"Invalid error table version: {version}, must be 2 or 3" + raise ValueError(msg) + + +def get_error_table(source) -> QTable: + """ + Fetches the error table from a SSW mirror or uses a local file if one is + provided. + + Parameters + ---------- + source : pathlib.Path, int + If input is a pathlib.Path, it is assumed to be a file path to a local error table. + If a path is provided as a string, it will error as an invalid source. + If the input is an int, it is assumed to be a version of the error table (2, 3). + + Returns + ------- + QTable + Error table. + + Raises + ------ + TypeError + If ``error_table`` is not a file path. + """ + if isinstance(source, pathlib.Path): + error_table = QTable(astropy.io.ascii.read(source)) + elif source in [2, 3]: + error_table = QTable(astropy.io.ascii.read(_fetch_error_table(source))) else: - msg = f"error_table must be a file path, an existing table, or None, not {type(error_table)}" + msg = f"``source`` must be a file path, or 2 or 3, not {source}" raise TypeError(msg) - table = QTable(table) - table["DATE"] = Time(table["DATE"], scale="utc") - table["T_START"] = Time(table["T_START"], scale="utc") + error_table["DATE"] = Time(error_table["DATE"], scale="utc") + error_table["T_START"] = Time(error_table["T_START"], scale="utc") # NOTE: The warning from erfa here is due to the fact that dates in # this table include at least one date from 2030 and converting this # date to UTC is ambiguous as the UTC conversion is not well defined # at this date. with warnings.catch_warnings(): warnings.simplefilter("ignore", category=ErfaWarning) - table["T_STOP"] = Time(table["T_STOP"], scale="utc") - table["WAVELNTH"] = u.Quantity(table["WAVELNTH"], "Angstrom") - table["DNPERPHT"] = u.Quantity(table["DNPERPHT"], "DN photon-1") - return table - - -@manager.require("error_table", *URL_HASH[ERROR_VERSION]) -def fetch_error_table(): - return manager.get("error_table") + error_table["T_STOP"] = Time(error_table["T_STOP"], scale="utc") + error_table["WAVELNTH"] = u.Quantity(error_table["WAVELNTH"], "Angstrom") + error_table["DNPERPHT"] = u.Quantity(error_table["DNPERPHT"], "DN photon-1") + return error_table diff --git a/aiapy/conftest.py b/aiapy/conftest.py index bc560ec..b2d040e 100644 --- a/aiapy/conftest.py +++ b/aiapy/conftest.py @@ -8,13 +8,10 @@ import sunpy.map from sunpy import log -ALL_CHANNELS = (94, 131, 171, 193, 211, 304, 335, 1600, 1700, 4500) * u.angstrom -CHANNELS = (94, 131, 171, 193, 211, 304, 335) * u.angstrom - -# Force MPL to use non-gui backends for testing. with contextlib.suppress(ImportError): import matplotlib as mpl + # Force MPL to use non-gui backends for testing. mpl.use("Agg") @@ -25,16 +22,6 @@ def aia_171_map(): return m.resample((4096, 4096) * u.pixel) -@pytest.fixture -def all_channels(): - return CHANNELS - - -@pytest.fixture -def channels(): - return CHANNELS - - @pytest.fixture def psf_94(channels): import aiapy.psf @@ -47,7 +34,7 @@ def idl_available() -> bool | None: import hissw hissw.Environment().run("") - return True + return True # NOQA: TRY300 except Exception as e: # NOQA: BLE001 log.warning(e) return False diff --git a/aiapy/util/__init__.py b/aiapy/util/__init__.py index 677e273..f3e6a1e 100644 --- a/aiapy/util/__init__.py +++ b/aiapy/util/__init__.py @@ -2,5 +2,7 @@ Subpackage with miscellaneous utility functions. """ +from .decorators import * from .exceptions import * +from .net import * from .util import * diff --git a/aiapy/util/net.py b/aiapy/util/net.py new file mode 100644 index 0000000..e881ab2 --- /dev/null +++ b/aiapy/util/net.py @@ -0,0 +1,49 @@ +""" +Provides basic functionality for querying the JSOC for internal use. +""" + +import drms + +__all__ = ["get_data_from_jsoc"] + + +def get_data_from_jsoc(query, *, key, seg=None): + """ + A simple wrapper around `~drms.Client.query` that raises a more informative + error message. + + This was created to avoid having to write a try/except block for every call to the JSOC within the package. + + Please note that this function is not intended to be used outside of the package. + + Parameters + ---------- + query : str + Query string to be passed to `~drms.Client.query`. + key : str + Key to be passed to `~drms.Client.query`. + seg : str + Seg to be passed to `~drms.Client.query`. + + Returns + ------- + pandas.DataFrame + The results of the query. + + Raises + ------ + OSError + If the query fails for any reason. + """ + try: + return drms.Client().query(query, key=key, seg=seg) + except KeyError as e: + # This probably should not be here but yolo. + # If there's no pointing information available between these times, + # JSOC will raise a cryptic KeyError + # (see https://github.com/LM-SAL/aiapy/issues/71) + msg = "Could not find any pointing information" + raise RuntimeError(msg) from e + except Exception as e: + msg = f"Unable to query the JSOC.\n Error message: {e}" + raise OSError(msg) from e diff --git a/aiapy/util/tests/test_net.py b/aiapy/util/tests/test_net.py new file mode 100644 index 0000000..d06c595 --- /dev/null +++ b/aiapy/util/tests/test_net.py @@ -0,0 +1,14 @@ +import pytest + +from aiapy.util.net import get_data_from_jsoc + + +@pytest.mark.remote_data +@pytest.mark.xfail(reason="JSOC is currently down") +def test_get_data_from_jsoc(): + assert get_data_from_jsoc("aia.lev1[2001-01-01]", key="T_OBS") is not None + + +def test_get_data_from_jsoc_error(): + with pytest.raises(OSError, match="Unable to query the JSOC."): + get_data_from_jsoc("aia.lev1[2001-01-01]", key="T_OBS") diff --git a/aiapy/util/util.py b/aiapy/util/util.py index 457ef48..2ba4b88 100644 --- a/aiapy/util/util.py +++ b/aiapy/util/util.py @@ -8,10 +8,10 @@ from astropy.coordinates import SkyCoord from astropy.time import Time -import drms from sunpy.time import parse_time from aiapy.util.decorators import validate_channel +from aiapy.util.net import get_data_from_jsoc __all__ = ["sdo_location", "telescope_number"] @@ -36,12 +36,9 @@ def sdo_location(time): """ t = parse_time(time) # Query for +/- 3 seconds around the given time - keys = drms.Client().query( - f"aia.lev1[{(t - 3*u.s).utc.isot}/6s]", - key="T_OBS, HAEX_OBS, HAEY_OBS, HAEZ_OBS", - ) + keys = get_data_from_jsoc(query=f"aia.lev1[{(t - 3*u.s).utc.isot}/6s]", key="T_OBS, HAEX_OBS, HAEY_OBS, HAEZ_OBS") if keys is None or len(keys) == 0: - msg = "No DRMS records near this time" + msg = f"No JSOC records near this time: {t}" raise ValueError(msg) # Linear interpolation between the nearest records within the returned set times = Time(list(keys["T_OBS"]), scale="utc") diff --git a/examples/correct_degradation.py b/examples/skip_correct_degradation.py similarity index 100% rename from examples/correct_degradation.py rename to examples/skip_correct_degradation.py diff --git a/examples/download_specific_data.py b/examples/skip_download_specific_data.py similarity index 100% rename from examples/download_specific_data.py rename to examples/skip_download_specific_data.py diff --git a/examples/replace_hot_pixels.py b/examples/skip_replace_hot_pixels.py similarity index 100% rename from examples/replace_hot_pixels.py rename to examples/skip_replace_hot_pixels.py diff --git a/ruff.toml b/ruff.toml index e21c779..6770541 100644 --- a/ruff.toml +++ b/ruff.toml @@ -74,8 +74,13 @@ lint.extend-ignore = [ ] "*conftest.py" = [ "D100", # Missing docstring in public module + "D103", # Missing docstring in public function "D104", # Missing docstring in public package ] +# TODO: Fix +"aiapy/response/channel.py" = [ + "D102", # Missing docstring in public method +] [lint.pydocstyle] convention = "numpy" From 0aff2e8855fbbbee5691346b48622fdc72e098de Mon Sep 17 00:00:00 2001 From: Nabil Freij Date: Tue, 31 Dec 2024 19:59:37 -0800 Subject: [PATCH 02/11] Offline tests --- aiapy/calibrate/tests/test_prep.py | 39 +++++---------- aiapy/calibrate/tests/test_uncertainty.py | 5 +- aiapy/calibrate/tests/test_util.py | 34 ++++++------- aiapy/calibrate/uncertainty.py | 27 ++++------- aiapy/calibrate/util.py | 58 ++++++++++------------- aiapy/conftest.py | 5 ++ aiapy/psf/tests/conftest.py | 5 +- aiapy/psf/tests/test_psf.py | 7 +-- aiapy/response/channel.py | 18 ++----- aiapy/response/tests/test_channel.py | 20 ++------ aiapy/tests/data/__init__.py | 2 +- aiapy/util/tests/test_net.py | 4 +- examples/instrument_degradation.py | 8 ++-- examples/skip_download_specific_data.py | 2 +- 14 files changed, 88 insertions(+), 146 deletions(-) diff --git a/aiapy/calibrate/tests/test_prep.py b/aiapy/calibrate/tests/test_prep.py index 04f5d6c..d69677b 100644 --- a/aiapy/calibrate/tests/test_prep.py +++ b/aiapy/calibrate/tests/test_prep.py @@ -100,26 +100,23 @@ def test_register_level_15(lvl_15_map) -> None: @pytest.mark.parametrize( - ("correction_table", "version"), + ("source"), [ - pytest.param(None, None, marks=pytest.mark.remote_data), - ( - get_correction_table(correction_table=get_test_filepath("aia_V8_20171210_050627_response_table.txt")), - 8, - ), + pytest.param("jsoc", marks=pytest.mark.remote_data), + pytest.param(8, marks=pytest.mark.remote_data), + get_test_filepath("aia_V8_20171210_050627_response_table.txt"), ], ) -def test_correct_degradation(aia_171_map, correction_table, version) -> None: +def test_correct_degradation(aia_171_map, source) -> None: + correction_table = get_correction_table(source=source) original_corrected = correct_degradation( aia_171_map, correction_table=correction_table, - calibration_version=version, ) d = degradation( aia_171_map.wavelength, aia_171_map.date, correction_table=correction_table, - calibration_version=version, ) with np.errstate(divide="ignore", invalid="ignore"): uncorrected_over_corrected = aia_171_map.data / original_corrected.data @@ -129,50 +126,39 @@ def test_correct_degradation(aia_171_map, correction_table, version) -> None: @pytest.mark.parametrize( - ("correction_table", "version", "time_correction_truth"), + ("source", "time_correction_truth"), [ pytest.param( - None, 10, 0.9031773242843387 * u.dimensionless_unscaled, marks=pytest.mark.remote_data, ), pytest.param( - None, 9, 0.8658650561969473 * u.dimensionless_unscaled, marks=pytest.mark.remote_data, ), pytest.param( - None, 8, 0.7667012041798814 * u.dimensionless_unscaled, marks=pytest.mark.remote_data, ), ( get_test_filepath("aia_V8_20171210_050627_response_table.txt"), - 8, - 0.7667108920899671 * u.dimensionless_unscaled, - ), - ( - get_correction_table(correction_table=get_test_filepath("aia_V8_20171210_050627_response_table.txt")), - 8, 0.7667108920899671 * u.dimensionless_unscaled, ), ], ) -def test_degradation(correction_table, version, time_correction_truth) -> None: +def test_degradation(source, time_correction_truth) -> None: # NOTE: this just tests an expected result from aiapy, not necessarily an # absolutely correct result. It was calculated for the above time and # the specific correction table file. - # NOTE: If the first test starts failing, it may be because the correction - # table parameters have been updated in JSOC. # TODO: Test this over multiple wavelengths + correction_table = get_correction_table(source=source) obstime = astropy.time.Time("2015-01-01T00:00:00", scale="utc") time_correction = degradation( 94 * u.angstrom, obstime, - calibration_version=version, correction_table=correction_table, ) assert u.allclose(time_correction, time_correction_truth, rtol=1e-10, atol=0.0) @@ -242,23 +228,22 @@ def test_degradation_4500() -> None: ): degradation(4500 * u.angstrom, obstime) - correction = degradation(4500 * u.angstrom, obstime, calibration_version=3) + correction = degradation(4500 * u.angstrom, obstime) assert u.allclose(correction, 1.0 * u.dimensionless_unscaled) def test_degradation_time_array() -> None: obstime = astropy.time.Time("2015-01-01T00:00:00", scale="utc") obstime = obstime + np.linspace(0, 1, 100) * u.year - correction_table = get_test_filepath("aia_V8_20171210_050627_response_table.txt") + correction_table = get_correction_table(get_test_filepath("aia_V8_20171210_050627_response_table.txt")) time_correction = degradation( 94 * u.angstrom, obstime, correction_table=correction_table, - calibration_version=8, ) assert time_correction.shape == obstime.shape for o, tc in zip(obstime, time_correction, strict=True): - assert tc == degradation(94 * u.angstrom, o, correction_table=correction_table, calibration_version=8) + assert tc == degradation(94 * u.angstrom, o, correction_table=correction_table) def test_register_cupy(aia_171_map) -> None: diff --git a/aiapy/calibrate/tests/test_uncertainty.py b/aiapy/calibrate/tests/test_uncertainty.py index de9ccb8..7a454c8 100644 --- a/aiapy/calibrate/tests/test_uncertainty.py +++ b/aiapy/calibrate/tests/test_uncertainty.py @@ -2,17 +2,14 @@ import numpy as np import pytest -from numpy.random import default_rng import astropy.units as u from aiapy.calibrate import estimate_error from aiapy.calibrate.util import get_error_table +from aiapy.conftest import CHANNELS, RANDOM_GENERATOR from aiapy.tests.data import get_test_filepath -# These are not fixtures so that they can be easily used in the parametrize mark -RANDOM_GENERATOR = default_rng() -CHANNELS = [94, 131, 171, 193, 211, 304, 335, 1600, 1700, 4500] * u.angstrom table_local = get_error_table(get_test_filepath("aia_V3_error_table.txt")) diff --git a/aiapy/calibrate/tests/test_util.py b/aiapy/calibrate/tests/test_util.py index 17b7c80..b4fbba4 100644 --- a/aiapy/calibrate/tests/test_util.py +++ b/aiapy/calibrate/tests/test_util.py @@ -1,3 +1,5 @@ +import re + import pytest import astropy.table @@ -14,22 +16,18 @@ # These are not fixtures so that they can be easily used in the parametrize mark obstime = astropy.time.Time("2015-01-01T00:00:00", scale="utc") -correction_table_local = get_correction_table( - correction_table=get_test_filepath("aia_V8_20171210_050627_response_table.txt"), -) -error_table_local = get_error_table(get_test_filepath("aia_V3_error_table.txt")) +correction_table_local = get_correction_table(get_test_filepath("aia_V8_20171210_050627_response_table.txt")) @pytest.mark.parametrize( - "correction_table", + "source", [ pytest.param(None, marks=pytest.mark.remote_data), - correction_table_local, get_test_filepath("aia_V8_20171210_050627_response_table.txt"), ], ) -def test_correction_table(correction_table) -> None: - table = get_correction_table(correction_table=correction_table) +def test_correction_table(source) -> None: + table = get_correction_table(source=source) assert isinstance(table, astropy.table.QTable) expected_columns = [ "VER_NUM", @@ -49,7 +47,7 @@ def test_correction_table(correction_table) -> None: @pytest.mark.parametrize("wavelength", [94 * u.angstrom, 1600 * u.angstrom]) def test_correction_table_selection(wavelength) -> None: - table = _select_epoch_from_correction_table(wavelength, obstime, correction_table_local, version=8) + table = _select_epoch_from_correction_table(wavelength, obstime, correction_table_local) assert isinstance(table, astropy.table.QTable) expected_columns = [ "VER_NUM", @@ -70,9 +68,11 @@ def test_correction_table_selection(wavelength) -> None: def test_invalid_correction_table_input() -> None: with pytest.raises( ValueError, - match="correction_table must be a file path, an existing table, or None.", + match=re.escape( + "correction_table must be a file path (pathlib.Path), 'jsoc' or one of 3, 4, 6, 7, 8, 9, 10. Not -1" + ), ): - get_correction_table(correction_table=-1) + get_correction_table(source=-1) def test_invalid_wavelength_raises_exception() -> None: @@ -80,15 +80,10 @@ def test_invalid_wavelength_raises_exception() -> None: _select_epoch_from_correction_table(1800 * u.angstrom, obstime, correction_table_local) -def test_wrong_version_number_raises_exception() -> None: - with pytest.raises(ValueError, match="Correction table does not contain calibration for version -1"): - _select_epoch_from_correction_table(94 * u.angstrom, obstime, correction_table_local, version=-1) - - def test_obstime_out_of_range() -> None: obstime_out_of_range = astropy.time.Time("2000-01-01T12:00:00", scale="utc") with pytest.raises(ValueError, match=f"No valid calibration epoch for {obstime_out_of_range}"): - _select_epoch_from_correction_table(94 * u.angstrom, obstime_out_of_range, correction_table_local, version=8) + _select_epoch_from_correction_table(94 * u.angstrom, obstime_out_of_range, correction_table_local) @pytest.mark.remote_data @@ -125,7 +120,6 @@ def test_pointing_table_unavailable() -> None: [ pytest.param(None, marks=pytest.mark.remote_data), get_test_filepath("aia_V3_error_table.txt"), - error_table_local, ], ) def test_error_table(error_table) -> None: @@ -135,5 +129,5 @@ def test_error_table(error_table) -> None: def test_invalid_error_table_input() -> None: - with pytest.raises(TypeError, match="error_table must be a file path, an existing table, or None"): - get_error_table(error_table=-1) + with pytest.raises(TypeError, match="source must be a file path, or 2 or 3, not -1"): + get_error_table(-1) diff --git a/aiapy/calibrate/uncertainty.py b/aiapy/calibrate/uncertainty.py index 6d5255d..8f7df8e 100644 --- a/aiapy/calibrate/uncertainty.py +++ b/aiapy/calibrate/uncertainty.py @@ -8,7 +8,6 @@ from aiapy.util import telescope_number from aiapy.util.decorators import validate_channel -from .util import get_error_table __all__ = ["estimate_error"] @@ -23,7 +22,7 @@ def estimate_error( include_preflight=False, include_eve=False, include_chianti=False, - error_table=None, + error_table, **kwargs, ) -> u.DN / u.pix: """ @@ -34,9 +33,10 @@ def estimate_error( onboard compression. The calculation can also optionally include contributions from the photometric calibration and errors in the atomic data. - .. note:: This function is adapted directly from the - `aia_bp_estimate_error.pro `_ - routine in SolarSoft. + .. note:: + This function is adapted directly from the + `aia_bp_estimate_error.pro `__ + routine in SolarSoft. Parameters ---------- @@ -53,11 +53,9 @@ def estimate_error( Use the EVE photometric calibration. If True, ``include_preflight`` must be False. include_chianti : `bool`, optional If True, include the atomic data errors from CHIANTI in the uncertainty. - error_table : `~astropy.table.QTable` or path-like, optional - Error table to use. Can be an existing table or a path to a file. If an error table - is not specified, the latest version will be downloaded from SolarSoft. Once you've - downloaded this once, you won't need to download it again unless the remote version - changes. + error_table : `~astropy.table.QTable` + Error table to use. Use `~aiapy.calibrate.util.get_error_table` to get the + appropriate error table. Returns ------- @@ -68,21 +66,17 @@ def estimate_error( aiapy.calibrate.util.get_error_table """ counts = np.atleast_1d(counts) - error_table = get_error_table(error_table=error_table) error_table = error_table[error_table["WAVELNTH"] == channel] - # Shot noise # NOTE: pixel and photon are "unitless" so we multiply/divide by these # units such that the shot noise has the same units as counts pix_per_photon = 1 * u.pixel / u.photon # use this to get units right n_photon = counts / error_table["DNPERPHT"] * pix_per_photon shot = np.sqrt(n_photon) * error_table["DNPERPHT"] / np.sqrt(n_sample) / pix_per_photon - # Dark noise # NOTE: The dark error of 0.18 is from an analysis of long-term trends in the residual # dark error from 2015. dark = 0.18 * u.DN / u.pix - # Read noise if kwargs.get("compare_idl", False): # The IDL version hardcodes the read noise as 1.15 DN / pixel so we @@ -101,7 +95,6 @@ def estimate_error( 4: 1.14 * u.DN / u.pix, }[telescope_number(channel)] read = read_noise / np.sqrt(n_sample) - # Quantization # NOTE: The 1/sqrt(12) factor is the RMS error due to quantization. Under the assumption that the # signal is much larger than the least significant bit (LSB), the quantization is not correlated @@ -110,13 +103,11 @@ def estimate_error( # See https://en.wikipedia.org/wiki/Quantization_(signal_processing) quant_rms = (1 / np.sqrt(12)) * u.DN / u.pix quant = quant_rms / np.sqrt(n_sample) - # Onboard compression compress = shot / error_table["COMPRESS"] compress[compress < quant_rms] = quant_rms compress[counts < 25 * counts.unit] = 0 * counts.unit compress /= np.sqrt(n_sample) - # Photometric calibration if include_eve and include_preflight: msg = "Cannot include both EVE and pre-flight correction." @@ -127,9 +118,7 @@ def estimate_error( elif include_preflight: calib = error_table["CALERR"] calib = calib * counts - # CHIANTI atomic errors chianti = error_table["CHIANTI"] if include_chianti else 0 chianti = chianti * counts - return np.sqrt(shot**2 + dark**2 + read**2 + quant**2 + compress**2 + chianti**2 + calib**2) diff --git a/aiapy/calibrate/util.py b/aiapy/calibrate/util.py index ecceb23..a4244b6 100644 --- a/aiapy/calibrate/util.py +++ b/aiapy/calibrate/util.py @@ -23,39 +23,29 @@ from aiapy.util.net import get_data_from_jsoc __all__ = [ - "CALIBRATION_VERSION", - "ERROR_VERSION", - "URL_HASH_ERROR_TABLE", - "URL_HASH_POINTING_TABLE", - "URL_HASH_RESPONSE_TABLE", "get_correction_table", "get_error_table", "get_pointing_table", ] -# Default version of the degradation calibration curve to use. -# This needs to be incremented as the calibration is updated in JSOC. -CALIBRATION_VERSION = 10 # Error table filename available from SSW -AIA_ERROR_FILE = "sdo/aia/response/aia_V{}_error_table.txt" -# Most recent version number for error tables; increment as new versions become available -ERROR_VERSION = 3 +_AIA_ERROR_FILE = "sdo/aia/response/aia_V{}_error_table.txt" # URLs and SHA-256 hashes for each version of the error tables -URL_HASH_ERROR_TABLE = { +_URL_HASH_ERROR_TABLE = { 2: ( - [urljoin(mirror, AIA_ERROR_FILE.format(2)) for mirror in _SSW_MIRRORS], + [urljoin(mirror, _AIA_ERROR_FILE.format(2)) for mirror in _SSW_MIRRORS], "ac97ccc48057809723c27e3ef290c7d78ee35791d9054b2188baecfb5c290d0a", ), 3: ( - [urljoin(mirror, AIA_ERROR_FILE.format(3)) for mirror in _SSW_MIRRORS], + [urljoin(mirror, _AIA_ERROR_FILE.format(3)) for mirror in _SSW_MIRRORS], "66ff034923bb0fd1ad20e8f30c7d909e1a80745063957dd6010f81331acaf894", ), } -URL_HASH_POINTING_TABLE = ( +_URL_HASH_POINTING_TABLE = ( "https://aia.lmsal.com/public/master_aia_pointing3h.csv", "a2c80fa0ea3453c62c91f51df045ae04b771d5cbb51c6495ed56de0da2a5482e", ) -URL_HASH_RESPONSE_TABLE = { +_URL_HASH_RESPONSE_TABLE = { 10: ( [urljoin(mirror, "sdo/aia/response/aia_V10_20201119_190000_response_table.txt") for mirror in _SSW_MIRRORS], "0a3f2db39d05c44185f6fdeec928089fb55d1ce1e0a805145050c6356cbc6e98", @@ -94,46 +84,46 @@ def _fetch_response_table(version: int): # Until the delayed feature from sunpy (v6.1) is out, this function # will need to be like this. - if version not in URL_HASH_RESPONSE_TABLE: + if version not in _URL_HASH_RESPONSE_TABLE: msg = f"Invalid response table version: {version}" raise ValueError(msg) - @manager.require("response_table_v10", *URL_HASH_RESPONSE_TABLE[10]) + @manager.require("response_table_v10", *_URL_HASH_RESPONSE_TABLE[10]) def fetch_response_table_v10(): return manager.get("response_table_v10") - @manager.require("response_table_v9", *URL_HASH_RESPONSE_TABLE[9]) + @manager.require("response_table_v9", *_URL_HASH_RESPONSE_TABLE[9]) def fetch_response_table_v9(): return manager.get("response_table_v9") - @manager.require("response_table_v8", *URL_HASH_RESPONSE_TABLE[8]) + @manager.require("response_table_v8", *_URL_HASH_RESPONSE_TABLE[8]) def fetch_response_table_v8(): return manager.get("response_table_v8") - @manager.require("response_table_v7", *URL_HASH_RESPONSE_TABLE[7]) + @manager.require("response_table_v7", *_URL_HASH_RESPONSE_TABLE[7]) def fetch_response_table_v7(): return manager.get("response_table_v7") - @manager.require("response_table_v6", *URL_HASH_RESPONSE_TABLE[6]) + @manager.require("response_table_v6", *_URL_HASH_RESPONSE_TABLE[6]) def fetch_response_table_v6(): return manager.get("response_table_v6") - @manager.require("response_table_v4", *URL_HASH_RESPONSE_TABLE[4]) + @manager.require("response_table_v4", *_URL_HASH_RESPONSE_TABLE[4]) def fetch_response_table_v4(): return manager.get("response_table_v4") - @manager.require("response_table_v3", *URL_HASH_RESPONSE_TABLE[3]) + @manager.require("response_table_v3", *_URL_HASH_RESPONSE_TABLE[3]) def fetch_response_table_v3(): return manager.get("response_table_v3") - @manager.require("response_table_v2", *URL_HASH_RESPONSE_TABLE[2]) + @manager.require("response_table_v2", *_URL_HASH_RESPONSE_TABLE[2]) def fetch_response_table_v2(): return manager.get("response_table_v2") return locals()[f"fetch_response_table_v{version}"]() -def get_correction_table(*, source): +def get_correction_table(source): """ Return table of degradation correction factors. @@ -164,9 +154,9 @@ def get_correction_table(*, source): """ if isinstance(source, pathlib.Path): table = QTable(astropy.io.ascii.read(source)) - elif source in URL_HASH_RESPONSE_TABLE: + elif source in _URL_HASH_RESPONSE_TABLE: table = QTable(astropy.io.ascii.read(_fetch_response_table(source))) - elif source.lower() == "jsoc": + elif isinstance(source, str) and source.lower() == "jsoc": # NOTE: the [!1=1!] disables the drms PrimeKey logic and enables # the query to find records that are ordinarily considered # identical because the PrimeKeys for this series are WAVE_STR @@ -174,7 +164,9 @@ def get_correction_table(*, source): # latest record for each unique combination of those keywords. table = QTable.from_pandas(get_data_from_jsoc(query="aia.response[][!1=1!]", key="**ALL**")) else: - msg = "correction_table must be a file path (pathlib.Path), 'jsoc' or one of 3, 4, 6, 7, 8, 9, 10. Not {source}" + msg = ( + f"correction_table must be a file path (pathlib.Path), 'jsoc' or one of 3, 4, 6, 7, 8, 9, 10. Not {source}" + ) raise ValueError(msg) selected_cols = [ "DATE", @@ -245,7 +237,7 @@ def _select_epoch_from_correction_table(channel: u.angstrom, obstime, table): return QTable(table[[0, i_epoch[-1]]]) -@manager.require("pointing_table", *URL_HASH_POINTING_TABLE) +@manager.require("pointing_table", *_URL_HASH_POINTING_TABLE) def fetch_pointing_table(): manager.get("pointing_table") @@ -319,11 +311,11 @@ def get_pointing_table(start, end, *, source): def _fetch_error_table(version: int): # Until the delayed feature from sunpy (v6.1) is out, this function # will need to be like this. - @manager.require("error_table_v2", *URL_HASH_ERROR_TABLE[2]) + @manager.require("error_table_v2", *_URL_HASH_ERROR_TABLE[2]) def fetch_error_table_v2(): return manager.get("error_table_v2") - @manager.require("error_table_v3", *URL_HASH_ERROR_TABLE[3]) + @manager.require("error_table_v3", *_URL_HASH_ERROR_TABLE[3]) def fetch_error_table_v3(): return manager.get("error_table_v3") @@ -362,7 +354,7 @@ def get_error_table(source) -> QTable: elif source in [2, 3]: error_table = QTable(astropy.io.ascii.read(_fetch_error_table(source))) else: - msg = f"``source`` must be a file path, or 2 or 3, not {source}" + msg = f"source must be a file path, or 2 or 3, not {source}" raise TypeError(msg) error_table["DATE"] = Time(error_table["DATE"], scale="utc") error_table["T_START"] = Time(error_table["T_START"], scale="utc") diff --git a/aiapy/conftest.py b/aiapy/conftest.py index b2d040e..04a734a 100644 --- a/aiapy/conftest.py +++ b/aiapy/conftest.py @@ -1,6 +1,7 @@ import contextlib import pytest +from numpy.random import default_rng import astropy.units as u @@ -8,6 +9,10 @@ import sunpy.map from sunpy import log +RANDOM_GENERATOR = default_rng() +CHANNELS = [94, 131, 171, 193, 211, 304, 335] * u.angstrom +ALL_CHANNELS = [94, 131, 171, 193, 211, 304, 335, 1600, 1700, 4500] * u.angstrom + with contextlib.suppress(ImportError): import matplotlib as mpl diff --git a/aiapy/psf/tests/conftest.py b/aiapy/psf/tests/conftest.py index 8084f75..286f995 100644 --- a/aiapy/psf/tests/conftest.py +++ b/aiapy/psf/tests/conftest.py @@ -5,8 +5,9 @@ import pytest import aiapy.psf +from aiapy.conftest import CHANNELS @pytest.fixture -def psf(channels): - return aiapy.psf.psf(channels[0], use_preflightcore=True, diffraction_orders=[-1, 0, 1]) +def psf(): + return aiapy.psf.psf(CHANNELS[0], use_preflightcore=True, diffraction_orders=[-1, 0, 1]) diff --git a/aiapy/psf/tests/test_psf.py b/aiapy/psf/tests/test_psf.py index 7192c5a..456761e 100644 --- a/aiapy/psf/tests/test_psf.py +++ b/aiapy/psf/tests/test_psf.py @@ -2,6 +2,7 @@ import pytest import aiapy.psf +from aiapy.conftest import CHANNELS MESH_PROPERTIES = [ "angle_arm", @@ -15,11 +16,11 @@ @pytest.mark.parametrize("use_preflightcore", [True, False]) -def test_filter_mesh_parameters(use_preflightcore, channels) -> None: +def test_filter_mesh_parameters(use_preflightcore) -> None: params = aiapy.psf.filter_mesh_parameters(use_preflightcore=use_preflightcore) assert isinstance(params, dict) - assert all(c in params for c in channels) - assert all(all(p in params[c] for p in MESH_PROPERTIES) for c in channels) + assert all(c in params for c in CHANNELS) + assert all(all(p in params[c] for p in MESH_PROPERTIES) for c in CHANNELS) def test_psf(psf) -> None: diff --git a/aiapy/response/channel.py b/aiapy/response/channel.py index ad1f900..ef9cf1a 100644 --- a/aiapy/response/channel.py +++ b/aiapy/response/channel.py @@ -282,7 +282,7 @@ def crosstalk( return u.Quantity(np.zeros(self.wavelength.shape), u.cm**2) @u.quantity_input - def eve_correction(self, obstime, **kwargs) -> u.dimensionless_unscaled: + def eve_correction(self, obstime, source) -> u.dimensionless_unscaled: r""" Correct effective area to give good agreement with full-disk EVE data. @@ -306,17 +306,8 @@ def eve_correction(self, obstime, **kwargs) -> u.dimensionless_unscaled: ---------- obstime : `~astropy.time.Time` The time of the observation. - correction_table : `~astropy.table.Table` or `str`, optional - Table of correction parameters or path to correction table file. - If not specified, it will be queried from JSOC. - If you are calling this function repeatedly, it is recommended to - read the correction table once and pass it with this argument to avoid - multiple redundant network calls. - calibration_version : `int`, optional - The version of the calibration to use when calculating the - degradation. By default, this is the most recent version available - from JSOC. If you are using a specific calibration response file, - you may need to specify this according to the version in that file. + source : `pathlib.Path` or `str` or `int`, optional + TODO: UPDATE Returns ------- @@ -329,8 +320,7 @@ def eve_correction(self, obstime, **kwargs) -> u.dimensionless_unscaled: table = _select_epoch_from_correction_table( self.channel, obstime, - get_correction_table(correction_table=kwargs.get("correction_table")), - version=kwargs.get("calibration_version"), + get_correction_table(source=source), ) effective_area_interp = np.interp(table["EFF_WVLN"][-1], self.wavelength, self.effective_area) return table["EFF_AREA"][0] / effective_area_interp diff --git a/aiapy/response/tests/test_channel.py b/aiapy/response/tests/test_channel.py index 2f82958..aae4e8b 100644 --- a/aiapy/response/tests/test_channel.py +++ b/aiapy/response/tests/test_channel.py @@ -111,33 +111,25 @@ def test_effective_area(channel) -> None: @pytest.mark.parametrize( - ("correction_table", "version", "eve_correction_truth"), + ("source", "eve_correction_truth"), [ pytest.param( - None, 9, 0.9494731307817633 * u.dimensionless_unscaled, marks=pytest.mark.remote_data, ), pytest.param( - None, 8, 1.0140518082508945 * u.dimensionless_unscaled, marks=pytest.mark.remote_data, ), ( get_test_filepath("aia_V8_20171210_050627_response_table.txt"), - 8, - 1.0140386988603103 * u.dimensionless_unscaled, - ), - ( - get_correction_table(correction_table=get_test_filepath("aia_V8_20171210_050627_response_table.txt")), - 8, 1.0140386988603103 * u.dimensionless_unscaled, ), ], ) -def test_eve_correction(channel, correction_table, version, eve_correction_truth) -> None: +def test_eve_correction(channel, source, eve_correction_truth) -> None: # NOTE: this just tests an expected result from aiapy, not necessarily an # absolutely correct result. It was calculated for the above time and # the correction parameters in JSOC at the time this code was committed/ @@ -150,7 +142,8 @@ def test_eve_correction(channel, correction_table, version, eve_correction_truth # JSOC are not necessarily the same as those in the correction table files # in SSW though they should be close. obstime = astropy.time.Time("2015-01-01T00:00:00", scale="utc") - eve_correction = channel.eve_correction(obstime, correction_table=correction_table, calibration_version=version) + correction_table = get_correction_table(source=source) + eve_correction = channel.eve_correction(obstime, correction_table=correction_table) assert u.allclose(eve_correction, eve_correction_truth, rtol=1e-10, atol=0.0) @@ -166,13 +159,11 @@ def test_wavelength_response_no_idl(channel) -> None: channel.wavelength_response( obstime=astropy.time.Time.now(), correction_table=correction_table, - calibration_version=8, ) channel.wavelength_response( obstime=astropy.time.Time.now(), include_eve_correction=True, correction_table=correction_table, - calibration_version=8, ) @@ -198,12 +189,10 @@ def test_wavelength_response_no_crosstalk(channel, idl_environment) -> None: def test_wavelength_response_time(channel, idl_environment, include_eve_correction) -> None: now = astropy.time.Time.now() correction_table = get_test_filepath("aia_V8_20171210_050627_response_table.txt") - calibration_version = 8 r = channel.wavelength_response( obstime=now, include_eve_correction=include_eve_correction, correction_table=correction_table, - calibration_version=calibration_version, ) ssw = idl_environment.run( """ @@ -213,7 +202,6 @@ def test_wavelength_response_time(channel, idl_environment, include_eve_correcti save_vars=["r"], args={ "obstime": now.tai.isot, - "version": calibration_version, "evenorm": int(include_eve_correction), "respversion": "20171210_050627", }, diff --git a/aiapy/tests/data/__init__.py b/aiapy/tests/data/__init__.py index 3f21903..8f4a6f8 100644 --- a/aiapy/tests/data/__init__.py +++ b/aiapy/tests/data/__init__.py @@ -22,4 +22,4 @@ def get_test_filepath(filename, **kwargs): if isinstance(filename, Path): # NOTE: get_pkg_data_filename does not accept Path objects filename = filename.as_posix() - return get_pkg_data_filename(filename, package="aiapy.tests.data", **kwargs) + return Path(get_pkg_data_filename(filename, package="aiapy.tests.data", **kwargs)) diff --git a/aiapy/util/tests/test_net.py b/aiapy/util/tests/test_net.py index d06c595..00cc1fd 100644 --- a/aiapy/util/tests/test_net.py +++ b/aiapy/util/tests/test_net.py @@ -10,5 +10,5 @@ def test_get_data_from_jsoc(): def test_get_data_from_jsoc_error(): - with pytest.raises(OSError, match="Unable to query the JSOC."): - get_data_from_jsoc("aia.lev1[2001-01-01]", key="T_OBS") + with pytest.raises(OSError, match="Unable to query the JSOC"): + get_data_from_jsoc("abc", key="def") diff --git a/examples/instrument_degradation.py b/examples/instrument_degradation.py index e51fc15..ddb208f 100644 --- a/examples/instrument_degradation.py +++ b/examples/instrument_degradation.py @@ -38,11 +38,11 @@ # the table of correction parameters is publicly available via the # `Joint Science Operations Center (JSOC) `_. # -# First, fetch this correction table. It is not strictly necessary to do this explicitly, -# but will significantly speed up the calculation by only fetching the table -# once. +# First, fetch this correction table. We have to specify the source of the +# correction table. This can be either a local file or a version number of a +# file hosted in SSW or "jsoc" to fetch the latest version from JSOC. -correction_table = get_correction_table() +correction_table = get_correction_table(source=10) ############################################################################### # We want to compute the degradation for each EUV channel. diff --git a/examples/skip_download_specific_data.py b/examples/skip_download_specific_data.py index e073c0c..ca31855 100644 --- a/examples/skip_download_specific_data.py +++ b/examples/skip_download_specific_data.py @@ -60,7 +60,7 @@ # Otherwise you're making a call to the JSOC every single time. pointing_table = get_pointing_table(level_1_maps[0].date - 3 * u.h, level_1_maps[-1].date + 3 * u.h) # The same applies for the correction table. -correction_table = get_correction_table() +correction_table = get_correction_table(source=10) level_15_maps = [] for a_map in level_1_maps: From 028af4ad85f5863c9535afe3f09c404feaff5bc5 Mon Sep 17 00:00:00 2001 From: Nabil Freij Date: Tue, 31 Dec 2024 20:40:38 -0800 Subject: [PATCH 03/11] Online tests --- aiapy/calibrate/spikes.py | 2 +- aiapy/calibrate/tests/test_meta.py | 9 +++++--- aiapy/calibrate/tests/test_prep.py | 20 ++++++++++++----- aiapy/calibrate/tests/test_util.py | 35 ++++++++++++++++++++---------- aiapy/calibrate/util.py | 2 +- aiapy/util/tests/test_util.py | 2 +- 6 files changed, 47 insertions(+), 23 deletions(-) diff --git a/aiapy/calibrate/spikes.py b/aiapy/calibrate/spikes.py index c5ba07b..979c3f3 100644 --- a/aiapy/calibrate/spikes.py +++ b/aiapy/calibrate/spikes.py @@ -146,7 +146,7 @@ def fetch_spikes(smap, *, as_coords=False): Original intensity values of the spikes """ series = "aia.lev1_uv_24s" if smap.wavelength in (1600, 1700, 4500) * u.angstrom else "aia.lev1_euv_12s" - file = get_data_from_jsoc(f'{series}[{smap.date}/12s][WAVELNTH={smap.meta["wavelnth"]}]', seg="spikes") + file = get_data_from_jsoc(f'{series}[{smap.date}/12s][WAVELNTH={smap.meta["wavelnth"]}]', key=None, seg="spikes") _, spikes = fits.open(f'http://jsoc.stanford.edu{file["spikes"][0]}') # Loaded as floats, but they are actually integers spikes = spikes.data.astype(np.int32) diff --git a/aiapy/calibrate/tests/test_meta.py b/aiapy/calibrate/tests/test_meta.py index 0f0a8f7..5e53032 100644 --- a/aiapy/calibrate/tests/test_meta.py +++ b/aiapy/calibrate/tests/test_meta.py @@ -21,7 +21,7 @@ def test_fix_observer_location(aia_171_map) -> None: @pytest.fixture def pointing_table(aia_171_map): - return get_pointing_table(aia_171_map.date - 6 * u.h, aia_171_map.date + 6 * u.h) + return get_pointing_table(aia_171_map.date - 6 * u.h, aia_171_map.date + 6 * u.h, source="lmsal") @pytest.fixture @@ -52,7 +52,10 @@ def test_fix_pointing(aia_171_map, pointing_table) -> None: # Remove keys to at least test that they get set for k in keys: aia_171_map.meta.pop(k) - aia_map_updated = update_pointing(aia_171_map) + aia_map_updated = update_pointing( + aia_171_map, + pointing_table=get_pointing_table(aia_171_map.date - 6 * u.h, aia_171_map.date + 6 * u.h, source="lmsal"), + ) # FIXME: how do we check these values are accurate? assert all(k in aia_map_updated.meta for k in keys) # Check the case where we have specified the pointing @@ -109,7 +112,7 @@ def test_update_pointing_no_entry_raises_exception(aia_171_map, pointing_table) # This tests that an exception is thrown when entry corresponding to # T_START <= T_OBS < T_END cannot be found in the pointing table. # We explicitly set the T_OBS key - aia_171_map.meta["T_OBS"] = (aia_171_map.date + 1 * u.day).isot + aia_171_map.meta["T_OBS"] = (aia_171_map.date - 1000 * u.day).isot with pytest.raises(IndexError, match="No valid entries for"): update_pointing(aia_171_map, pointing_table=pointing_table) diff --git a/aiapy/calibrate/tests/test_prep.py b/aiapy/calibrate/tests/test_prep.py index d69677b..d8633fb 100644 --- a/aiapy/calibrate/tests/test_prep.py +++ b/aiapy/calibrate/tests/test_prep.py @@ -161,7 +161,7 @@ def test_degradation(source, time_correction_truth) -> None: obstime, correction_table=correction_table, ) - assert u.allclose(time_correction, time_correction_truth, rtol=1e-10, atol=0.0) + assert u.allclose(time_correction, time_correction_truth, atol=1e-3) @pytest.mark.parametrize( @@ -214,21 +214,29 @@ def test_degradation_all_wavelengths(wavelength, result) -> None: time_correction = degradation( wavelength * u.angstrom, obstime, + correction_table=get_correction_table(10), ) - assert u.allclose(time_correction, result) + assert u.allclose(time_correction, result, atol=1e-3) @pytest.mark.remote_data -def test_degradation_4500() -> None: +def test_degradation_4500_missing() -> None: # 4500 has a max version of 3, so by default it will error obstime = astropy.time.Time("2015-01-01T00:00:00", scale="utc") with pytest.raises( ValueError, - match="Correction table does not contain calibration for version 10 for 4500.0 Angstrom. Max version is 3", + match="Correction table does not contain calibration for 4500.0 Angstrom. Max version is 3.", ): - degradation(4500 * u.angstrom, obstime) + degradation(4500 * u.angstrom, obstime, correction_table=get_correction_table(10)) - correction = degradation(4500 * u.angstrom, obstime) + +@pytest.mark.xfail(reason="JSOC is down") +@pytest.mark.remote_data +def test_degradation_4500_jsoc() -> None: + # 4500 has a max version of 3, so by default it will error + # and it is missing from the SSW files but not the JSOC + obstime = astropy.time.Time("2015-01-01T00:00:00", scale="utc") + correction = degradation(4500 * u.angstrom, obstime, correction_table=get_correction_table("jsoc")) assert u.allclose(correction, 1.0 * u.dimensionless_unscaled) diff --git a/aiapy/calibrate/tests/test_util.py b/aiapy/calibrate/tests/test_util.py index b4fbba4..2d0e89d 100644 --- a/aiapy/calibrate/tests/test_util.py +++ b/aiapy/calibrate/tests/test_util.py @@ -22,7 +22,15 @@ @pytest.mark.parametrize( "source", [ - pytest.param(None, marks=pytest.mark.remote_data), + pytest.param("jsoc", marks=pytest.mark.remote_data), + pytest.param(3, marks=pytest.mark.remote_data), + pytest.param(4, marks=pytest.mark.remote_data), + pytest.param(5, marks=pytest.mark.remote_data), + pytest.param(6, marks=pytest.mark.remote_data), + pytest.param(7, marks=pytest.mark.remote_data), + pytest.param(8, marks=pytest.mark.remote_data), + pytest.param(9, marks=pytest.mark.remote_data), + pytest.param(10, marks=pytest.mark.remote_data), get_test_filepath("aia_V8_20171210_050627_response_table.txt"), ], ) @@ -87,6 +95,7 @@ def test_obstime_out_of_range() -> None: @pytest.mark.remote_data +@pytest.mark.xfail(reason="JSOC is down") def test_pointing_table() -> None: expected_columns = ["T_START", "T_STOP"] for c in ["094", "171", "193", "211", "304", "335", "1600", "1700", "4500"]: @@ -97,28 +106,32 @@ def test_pointing_table() -> None: f"A_{c}_IMSCALE", ] t = astropy.time.Time("2011-01-01T00:00:00", scale="utc") - table = get_pointing_table(t - 3 * u.h, t + 3 * u.h) - assert isinstance(table, astropy.table.QTable) - assert all(cn in table.colnames for cn in expected_columns) - assert isinstance(table["T_START"], astropy.time.Time) - assert isinstance(table["T_STOP"], astropy.time.Time) - # Ensure that none of the pointing parameters are masked columns - for c in expected_columns[2:]: - assert not hasattr(table[c], "mask") + table_lmsal = get_pointing_table(t - 3 * u.h, t + 3 * u.h, source="lmsal") + table_jsoc = get_pointing_table(t - 3 * u.h, t + 3 * u.h, source="jsoc") + for table in [table_lmsal, table_jsoc]: + assert isinstance(table, astropy.table.QTable) + assert all(cn in table.colnames for cn in expected_columns) + assert isinstance(table["T_START"], astropy.time.Time) + assert isinstance(table["T_STOP"], astropy.time.Time) + # Ensure that none of the pointing parameters are masked columns + for c in expected_columns[2:]: + assert not hasattr(table[c], "mask") @pytest.mark.remote_data +@pytest.mark.xfail(reason="JSOC is down") def test_pointing_table_unavailable() -> None: # Check that missing pointing data raises a nice error t = astropy.time.Time("1990-01-01") with pytest.raises(RuntimeError, match="Could not find any pointing information"): - get_pointing_table(t - 3 * u.h, t + 3 * u.h) + get_pointing_table(t - 3 * u.h, t + 3 * u.h, source="jsoc") @pytest.mark.parametrize( "error_table", [ - pytest.param(None, marks=pytest.mark.remote_data), + pytest.param(2, marks=pytest.mark.remote_data), + pytest.param(3, marks=pytest.mark.remote_data), get_test_filepath("aia_V3_error_table.txt"), ], ) diff --git a/aiapy/calibrate/util.py b/aiapy/calibrate/util.py index a4244b6..cfe6434 100644 --- a/aiapy/calibrate/util.py +++ b/aiapy/calibrate/util.py @@ -239,7 +239,7 @@ def _select_epoch_from_correction_table(channel: u.angstrom, obstime, table): @manager.require("pointing_table", *_URL_HASH_POINTING_TABLE) def fetch_pointing_table(): - manager.get("pointing_table") + return manager.get("pointing_table") def get_pointing_table(start, end, *, source): diff --git a/aiapy/util/tests/test_util.py b/aiapy/util/tests/test_util.py index 6a9df80..52bac19 100644 --- a/aiapy/util/tests/test_util.py +++ b/aiapy/util/tests/test_util.py @@ -16,5 +16,5 @@ def test_sdo_location(aia_171_map) -> None: @pytest.mark.remote_data def test_sdo_location_raises_error() -> None: # Confirm that an error is raised for a time without records - with pytest.raises(ValueError, match="No DRMS records near this time"): + with pytest.raises(ValueError, match="No JSOC records near this time: 2001-01-01T00:00:00.000"): aiapy.util.sdo_location("2001-01-01") From 6dd94b0c5eef53c91a232f0a10dbce4a7a8e9481 Mon Sep 17 00:00:00 2001 From: Nabil Freij Date: Fri, 3 Jan 2025 16:54:18 -0800 Subject: [PATCH 04/11] Docs --- .gitignore | 1 + aiapy/calibrate/util.py | 15 +++++++++------ aiapy/util/decorators.py | 5 +++-- changelog/346.breaking.rst | 22 ++++++++++++++++++++++ 4 files changed, 35 insertions(+), 8 deletions(-) create mode 100644 changelog/346.breaking.rst diff --git a/.gitignore b/.gitignore index 5f9d3d9..54b1486 100644 --- a/.gitignore +++ b/.gitignore @@ -74,6 +74,7 @@ instance/ # Sphinx documentation docs/_build/ +docs/generated/ # automodapi docs/api docs/sg_execution_times.rst diff --git a/aiapy/calibrate/util.py b/aiapy/calibrate/util.py index cfe6434..0b380a8 100644 --- a/aiapy/calibrate/util.py +++ b/aiapy/calibrate/util.py @@ -242,7 +242,7 @@ def fetch_pointing_table(): return manager.get("pointing_table") -def get_pointing_table(start, end, *, source): +def get_pointing_table(source, *, start=None, end=None): """ Retrieve 3-hourly master pointing table from the given source. @@ -268,14 +268,14 @@ def get_pointing_table(start, end, *, source): Parameters ---------- - start : `~astropy.time.Time` - Start time of the interval. - end : `~astropy.time.Time` - End time of the interval. source : str Name of the source from which to retrieve the pointing table. Must be one of ``"jsoc"`` or ``"lmsal"``. Note that the LMSAL pointing table is not updated frequently. + start : `~astropy.time.Time`, optional + Start time of the interval. Only required if ``source`` is ``"jsoc"``. + end : `~astropy.time.Time`, optional + End time of the interval. Only required if ``source`` is ``"jsoc"``. Returns ------- @@ -286,6 +286,9 @@ def get_pointing_table(start, end, *, source): aiapy.calibrate.update_pointing """ if source.lower() == "jsoc": + if start is None or end is None: + msg = "start and end must be provided if source is 'jsoc'" + raise ValueError(msg) table = get_data_from_jsoc(query=f"aia.master_pointing3h[{start.isot}Z-{end.isot}Z]", key="**ALL**") elif source.lower() == "lmsal": table = QTable(astropy_ascii.read(fetch_pointing_table())) @@ -341,7 +344,7 @@ def get_error_table(source) -> QTable: Returns ------- - QTable + `~astropy.table.QTable` Error table. Raises diff --git a/aiapy/util/decorators.py b/aiapy/util/decorators.py index 0b03abf..433e237 100644 --- a/aiapy/util/decorators.py +++ b/aiapy/util/decorators.py @@ -28,8 +28,9 @@ def validate_channel(argument, *, valid_channels="all"): ---------- argument : str Argument name to validate. - valid_channels : {'all'}, list - List of valid channels. If ``'all'``, validate against the list of all AIA channels. + valid_channels : list or str, optional + List of valid channels. + Defaults to "all", which will validate against the list of all AIA channels. """ if valid_channels == "all": valid_channels = _all_channels diff --git a/changelog/346.breaking.rst b/changelog/346.breaking.rst new file mode 100644 index 0000000..8a8a451 --- /dev/null +++ b/changelog/346.breaking.rst @@ -0,0 +1,22 @@ +Due to the temporary shutdown of the JSOC, many of the functions within ``aiapy`` are currently not functioning as they rely on the JSOC to retrieve data. + +In an effort to enable other sources of data to be provided to the broken functions, the following breaking changes have been made: + +1. `aiapy.calibrate.correct_degradation` has removed "calibration_version" keyword and now the "correction_table" is now a required argument. + To get a correction table, one can use the `aiapy.calibrate.util.get_correction_table`. + This allows one to select between, JSOC, SSW, or a custom correction table. + For SSW, the correction table is just a version of the SSW correction table which goes from 3 to 10. + If you want to use the JSOC, you can pass in "jsoc" as a string argument. + +2. `aiapy.calibrate.estimate_error`, the "error_table" keyword is now a required argument. + To get the error table, one can use the `aiapy.calibrate.util.get_error_table`. + There are only two options for the error table, 2 or 3. + +3. `aiapy.response.channel.Channel.eve_correction`, the correction_table keyword is now a required argument. + As for `aiapy.calibrate.correct_degradation`, one can use the `aiapy.calibrate.util.get_correction_table` to get the correction table. + +4. `aiapy.calibrate.update_pointing`, the "pointing_table" keyword is now a required argument. + To get the pointing table, one can use `aiapy.calibrate.util.get_pointing_table`. + This function now has a "source" keyword which can be used to select between JSOC and a copy of the pointing information dated from 11/20/2024 stored on LMSAL servers. + +Not that many of the files from SSW are not updated with any frequency and provide worse results than using the data from the JSOC. From 9e2c4d7bbb07c4ea4e548a8e8f16d8007d352131 Mon Sep 17 00:00:00 2001 From: Nabil Freij Date: Fri, 3 Jan 2025 17:44:21 -0800 Subject: [PATCH 05/11] Test tweaks --- aiapy/calibrate/tests/test_meta.py | 6 +++--- aiapy/calibrate/tests/test_util.py | 7 +++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/aiapy/calibrate/tests/test_meta.py b/aiapy/calibrate/tests/test_meta.py index 5e53032..954ab75 100644 --- a/aiapy/calibrate/tests/test_meta.py +++ b/aiapy/calibrate/tests/test_meta.py @@ -20,8 +20,8 @@ def test_fix_observer_location(aia_171_map) -> None: @pytest.fixture -def pointing_table(aia_171_map): - return get_pointing_table(aia_171_map.date - 6 * u.h, aia_171_map.date + 6 * u.h, source="lmsal") +def pointing_table(): + return get_pointing_table("lmsal") @pytest.fixture @@ -54,7 +54,7 @@ def test_fix_pointing(aia_171_map, pointing_table) -> None: aia_171_map.meta.pop(k) aia_map_updated = update_pointing( aia_171_map, - pointing_table=get_pointing_table(aia_171_map.date - 6 * u.h, aia_171_map.date + 6 * u.h, source="lmsal"), + pointing_table=get_pointing_table("lmsal"), ) # FIXME: how do we check these values are accurate? assert all(k in aia_map_updated.meta for k in keys) diff --git a/aiapy/calibrate/tests/test_util.py b/aiapy/calibrate/tests/test_util.py index 2d0e89d..31c4303 100644 --- a/aiapy/calibrate/tests/test_util.py +++ b/aiapy/calibrate/tests/test_util.py @@ -25,7 +25,6 @@ pytest.param("jsoc", marks=pytest.mark.remote_data), pytest.param(3, marks=pytest.mark.remote_data), pytest.param(4, marks=pytest.mark.remote_data), - pytest.param(5, marks=pytest.mark.remote_data), pytest.param(6, marks=pytest.mark.remote_data), pytest.param(7, marks=pytest.mark.remote_data), pytest.param(8, marks=pytest.mark.remote_data), @@ -106,8 +105,8 @@ def test_pointing_table() -> None: f"A_{c}_IMSCALE", ] t = astropy.time.Time("2011-01-01T00:00:00", scale="utc") - table_lmsal = get_pointing_table(t - 3 * u.h, t + 3 * u.h, source="lmsal") - table_jsoc = get_pointing_table(t - 3 * u.h, t + 3 * u.h, source="jsoc") + table_lmsal = get_pointing_table("lmsal") + table_jsoc = get_pointing_table("jsoc", start=t - 3 * u.h, end=t + 3 * u.h) for table in [table_lmsal, table_jsoc]: assert isinstance(table, astropy.table.QTable) assert all(cn in table.colnames for cn in expected_columns) @@ -124,7 +123,7 @@ def test_pointing_table_unavailable() -> None: # Check that missing pointing data raises a nice error t = astropy.time.Time("1990-01-01") with pytest.raises(RuntimeError, match="Could not find any pointing information"): - get_pointing_table(t - 3 * u.h, t + 3 * u.h, source="jsoc") + get_pointing_table("jsoc", start=t - 3 * u.h, end=t + 3 * u.h) @pytest.mark.parametrize( From eefb761938edf07005822f9be57f056e944bea4c Mon Sep 17 00:00:00 2001 From: Nabil Freij Date: Sat, 4 Jan 2025 13:33:26 -0800 Subject: [PATCH 06/11] Everything else --- .github/workflows/ci.yml | 4 ++-- aiapy/calibrate/tests/test_prep.py | 1 - aiapy/calibrate/tests/test_util.py | 4 +--- aiapy/calibrate/util.py | 10 +++++---- aiapy/response/channel.py | 22 +++++++++---------- aiapy/util/net.py | 13 +++++------ aiapy/util/tests/test_net.py | 3 +-- aiapy/util/tests/test_util.py | 2 +- examples/calculate_response_function.py | 14 ++++++++---- ..._degradation.py => correct_degradation.py} | 3 ++- examples/instrument_degradation.py | 2 +- examples/prepping_level_1_data.py | 11 +++++++--- ...ce_hot_pixels.py => replace_hot_pixels.py} | 0 examples/skip_download_specific_data.py | 4 ++-- examples/update_header_keywords.py | 14 +++++++----- pyproject.toml | 1 + tox.ini | 6 +++-- 17 files changed, 64 insertions(+), 50 deletions(-) rename examples/{skip_correct_degradation.py => correct_degradation.py} (96%) rename examples/{skip_replace_hot_pixels.py => replace_hot_pixels.py} (100%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 971f234..0f1db87 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -74,7 +74,7 @@ jobs: .tox/sample_data/ cache-key: docs-${{ github.run_id }} envs: | - - linux: build_docs + - linux: build-docs online: if: "!startsWith(github.event.ref, 'refs/tags/v')" @@ -86,7 +86,7 @@ jobs: coverage: codecov toxdeps: tox-pypi-filter envs: | - - linux: build_docs-gallery + - linux: build-docs-gallery pytest: false cache-path: | docs/_build/ diff --git a/aiapy/calibrate/tests/test_prep.py b/aiapy/calibrate/tests/test_prep.py index d8633fb..7a2949b 100644 --- a/aiapy/calibrate/tests/test_prep.py +++ b/aiapy/calibrate/tests/test_prep.py @@ -230,7 +230,6 @@ def test_degradation_4500_missing() -> None: degradation(4500 * u.angstrom, obstime, correction_table=get_correction_table(10)) -@pytest.mark.xfail(reason="JSOC is down") @pytest.mark.remote_data def test_degradation_4500_jsoc() -> None: # 4500 has a max version of 3, so by default it will error diff --git a/aiapy/calibrate/tests/test_util.py b/aiapy/calibrate/tests/test_util.py index 31c4303..159e8fc 100644 --- a/aiapy/calibrate/tests/test_util.py +++ b/aiapy/calibrate/tests/test_util.py @@ -94,7 +94,6 @@ def test_obstime_out_of_range() -> None: @pytest.mark.remote_data -@pytest.mark.xfail(reason="JSOC is down") def test_pointing_table() -> None: expected_columns = ["T_START", "T_STOP"] for c in ["094", "171", "193", "211", "304", "335", "1600", "1700", "4500"]: @@ -118,11 +117,10 @@ def test_pointing_table() -> None: @pytest.mark.remote_data -@pytest.mark.xfail(reason="JSOC is down") def test_pointing_table_unavailable() -> None: # Check that missing pointing data raises a nice error t = astropy.time.Time("1990-01-01") - with pytest.raises(RuntimeError, match="Could not find any pointing information"): + with pytest.raises(RuntimeError, match="No data found for this query"): get_pointing_table("jsoc", start=t - 3 * u.h, end=t + 3 * u.h) diff --git a/aiapy/calibrate/util.py b/aiapy/calibrate/util.py index 0b380a8..7aad333 100644 --- a/aiapy/calibrate/util.py +++ b/aiapy/calibrate/util.py @@ -198,7 +198,7 @@ def get_correction_table(source): @u.quantity_input @validate_channel("channel") -def _select_epoch_from_correction_table(channel: u.angstrom, obstime, table): +def _select_epoch_from_correction_table(channel: u.angstrom, obstime, correction_table): """ Return correction table with only the first epoch and the epoch in which ``obstime`` falls and for only one given calibration version. @@ -207,14 +207,14 @@ def _select_epoch_from_correction_table(channel: u.angstrom, obstime, table): ---------- channel : `~astropy.units.Quantity` obstime : `~astropy.time.Time` - table : `~astropy.table.QTable` + correction_table : `~astropy.table.QTable` """ # Select only this channel # NOTE: The WAVE_STR prime keys for the aia.response JSOC series for the # non-EUV channels do not have a thick/thin designation thin = "_THIN" if channel not in (1600, 1700, 4500) * u.angstrom else "" wave = channel.to(u.angstrom).value - table = table[table["WAVE_STR"] == f"{wave:.0f}{thin}"] + table = correction_table[correction_table["WAVE_STR"] == f"{wave:.0f}{thin}"] table.sort("DATE") # Newest entries will be last if len(table) == 0: extra_msg = " Max version is 3." if channel == 4500 * u.AA else "" @@ -289,7 +289,9 @@ def get_pointing_table(source, *, start=None, end=None): if start is None or end is None: msg = "start and end must be provided if source is 'jsoc'" raise ValueError(msg) - table = get_data_from_jsoc(query=f"aia.master_pointing3h[{start.isot}Z-{end.isot}Z]", key="**ALL**") + table = QTable.from_pandas( + get_data_from_jsoc(query=f"aia.master_pointing3h[{start.isot}Z-{end.isot}Z]", key="**ALL**") + ) elif source.lower() == "lmsal": table = QTable(astropy_ascii.read(fetch_pointing_table())) else: diff --git a/aiapy/response/channel.py b/aiapy/response/channel.py index ef9cf1a..a66e024 100644 --- a/aiapy/response/channel.py +++ b/aiapy/response/channel.py @@ -15,7 +15,7 @@ from aiapy import _SSW_MIRRORS from aiapy.calibrate import degradation -from aiapy.calibrate.util import _select_epoch_from_correction_table, get_correction_table +from aiapy.calibrate.util import _select_epoch_from_correction_table from aiapy.data._manager import manager from aiapy.util import telescope_number from aiapy.util.decorators import validate_channel @@ -282,7 +282,7 @@ def crosstalk( return u.Quantity(np.zeros(self.wavelength.shape), u.cm**2) @u.quantity_input - def eve_correction(self, obstime, source) -> u.dimensionless_unscaled: + def eve_correction(self, obstime, correction_table) -> u.dimensionless_unscaled: r""" Correct effective area to give good agreement with full-disk EVE data. @@ -306,8 +306,9 @@ def eve_correction(self, obstime, source) -> u.dimensionless_unscaled: ---------- obstime : `~astropy.time.Time` The time of the observation. - source : `pathlib.Path` or `str` or `int`, optional - TODO: UPDATE + correction_table : `astropy.table.QTable` + Table of correction parameters. + See `aiapy.calibrate.util.get_correction_table` for more information. Returns ------- @@ -320,7 +321,7 @@ def eve_correction(self, obstime, source) -> u.dimensionless_unscaled: table = _select_epoch_from_correction_table( self.channel, obstime, - get_correction_table(source=source), + correction_table=correction_table, ) effective_area_interp = np.interp(table["EFF_WVLN"][-1], self.wavelength, self.effective_area) return table["EFF_AREA"][0] / effective_area_interp @@ -360,7 +361,7 @@ def wavelength_response( obstime=None, include_eve_correction=False, include_crosstalk=True, - **kwargs, + correction_table, ) -> u.DN / u.ph * u.cm**2: r""" The wavelength response function is the product of the gain and the @@ -393,9 +394,8 @@ def wavelength_response( The time-dependent correction is also included. include_crosstalk : `bool`, optional If true, include the effect of crosstalk between channels that share a telescope - correction_table : `~astropy.table.Table` or `str`, optional - Table of correction parameters or path to correction table file. - If not specified, it will be queried from JSOC. + correction_table : `~astropy.table.Table` + Table of correction parameters. See `aiapy.calibrate.util.get_correction_table` for more information. Returns @@ -412,8 +412,8 @@ def wavelength_response( """ eve_correction, time_correction = 1, 1 if obstime is not None: - time_correction = degradation(self.channel, obstime, **kwargs) + time_correction = degradation(self.channel, obstime, correction_table=correction_table) if include_eve_correction: - eve_correction = self.eve_correction(obstime, **kwargs) + eve_correction = self.eve_correction(obstime, correction_table=correction_table) crosstalk = self.crosstalk if include_crosstalk else 0 * u.cm**2 return (self.effective_area + crosstalk) * self.gain * time_correction * eve_correction diff --git a/aiapy/util/net.py b/aiapy/util/net.py index e881ab2..3b579b0 100644 --- a/aiapy/util/net.py +++ b/aiapy/util/net.py @@ -36,14 +36,11 @@ def get_data_from_jsoc(query, *, key, seg=None): If the query fails for any reason. """ try: - return drms.Client().query(query, key=key, seg=seg) - except KeyError as e: - # This probably should not be here but yolo. - # If there's no pointing information available between these times, - # JSOC will raise a cryptic KeyError - # (see https://github.com/LM-SAL/aiapy/issues/71) - msg = "Could not find any pointing information" - raise RuntimeError(msg) from e + jsoc_result = drms.Client().query(query, key=key, seg=seg) except Exception as e: msg = f"Unable to query the JSOC.\n Error message: {e}" raise OSError(msg) from e + if len(jsoc_result) == 0: + msg = f"No data found for this query: {query}, key: {key}, seg: {seg}" + raise RuntimeError(msg) + return jsoc_result diff --git a/aiapy/util/tests/test_net.py b/aiapy/util/tests/test_net.py index 00cc1fd..1b5b576 100644 --- a/aiapy/util/tests/test_net.py +++ b/aiapy/util/tests/test_net.py @@ -4,9 +4,8 @@ @pytest.mark.remote_data -@pytest.mark.xfail(reason="JSOC is currently down") def test_get_data_from_jsoc(): - assert get_data_from_jsoc("aia.lev1[2001-01-01]", key="T_OBS") is not None + assert get_data_from_jsoc("aia.master_pointing3h[2010-05-13T00:00:00Z]", key="**ALL**") is not None def test_get_data_from_jsoc_error(): diff --git a/aiapy/util/tests/test_util.py b/aiapy/util/tests/test_util.py index 52bac19..1e24bd3 100644 --- a/aiapy/util/tests/test_util.py +++ b/aiapy/util/tests/test_util.py @@ -16,5 +16,5 @@ def test_sdo_location(aia_171_map) -> None: @pytest.mark.remote_data def test_sdo_location_raises_error() -> None: # Confirm that an error is raised for a time without records - with pytest.raises(ValueError, match="No JSOC records near this time: 2001-01-01T00:00:00.000"): + with pytest.raises(RuntimeError, match="No data found for this query"): aiapy.util.sdo_location("2001-01-01") diff --git a/examples/calculate_response_function.py b/examples/calculate_response_function.py index 0e9e7ee..0ce6a6e 100644 --- a/examples/calculate_response_function.py +++ b/examples/calculate_response_function.py @@ -14,6 +14,7 @@ import astropy.time import astropy.units as u +from aiapy.calibrate.util import get_correction_table from aiapy.response import Channel ############################################################################### @@ -94,7 +95,8 @@ # Additionally, `aiapy.response.Channel` provides a method for calculating # the wavelength response function using the equation above, -wavelength_response_335 = aia_335_channel.wavelength_response() +correction_table = get_correction_table("jsoc") +wavelength_response_335 = aia_335_channel.wavelength_response(correction_table=correction_table) print(wavelength_response_335) ############################################################################### @@ -118,7 +120,9 @@ # by default in the wavelength response calculation. To exclude this # effect, -wavelength_response_335_no_cross = aia_335_channel.wavelength_response(include_crosstalk=False) +wavelength_response_335_no_cross = aia_335_channel.wavelength_response( + include_crosstalk=False, correction_table=correction_table +) ############################################################################### # If we look at the response around 131 Å (the channel with which 335 Å shares @@ -145,8 +149,10 @@ # of 1 January 2019, obstime = astropy.time.Time("2019-01-01T00:00:00") -wavelength_response_335_time = aia_335_channel.wavelength_response(obstime=obstime) -wavelength_response_335_eve = aia_335_channel.wavelength_response(obstime=obstime, include_eve_correction=True) +wavelength_response_335_time = aia_335_channel.wavelength_response(obstime=obstime, correction_table=correction_table) +wavelength_response_335_eve = aia_335_channel.wavelength_response( + obstime=obstime, include_eve_correction=True, correction_table=correction_table +) ############################################################################### # We can then compare the two corrected response diff --git a/examples/skip_correct_degradation.py b/examples/correct_degradation.py similarity index 96% rename from examples/skip_correct_degradation.py rename to examples/correct_degradation.py index 48ed05c..05f627d 100644 --- a/examples/skip_correct_degradation.py +++ b/examples/correct_degradation.py @@ -16,6 +16,7 @@ from sunpy.net import attrs as a from aiapy.calibrate import degradation +from aiapy.calibrate.util import get_correction_table # This lets you pass `astropy.time.Time` objects directly to matplotlib time_support(format="jyear") @@ -51,7 +52,7 @@ # For more details on how the correction factor is calculated, see the documentation for the # `aiapy.calibrate.degradation` function. -correction_factor = degradation(335 * u.angstrom, table["DATE_OBS"]) +correction_factor = degradation(335 * u.angstrom, table["DATE_OBS"], correction_table=get_correction_table("jsoc")) table["DATAMEAN_DEG"] = table["DATAMEAN"] / correction_factor ############################################################################### diff --git a/examples/instrument_degradation.py b/examples/instrument_degradation.py index ddb208f..1398094 100644 --- a/examples/instrument_degradation.py +++ b/examples/instrument_degradation.py @@ -42,7 +42,7 @@ # correction table. This can be either a local file or a version number of a # file hosted in SSW or "jsoc" to fetch the latest version from JSOC. -correction_table = get_correction_table(source=10) +correction_table = get_correction_table("JSOC") ############################################################################### # We want to compute the degradation for each EUV channel. diff --git a/examples/prepping_level_1_data.py b/examples/prepping_level_1_data.py index ed789da..cb66909 100644 --- a/examples/prepping_level_1_data.py +++ b/examples/prepping_level_1_data.py @@ -12,10 +12,13 @@ import matplotlib.pyplot as plt +import astropy.units as u + import sunpy.map import aiapy.data.sample as sample_data from aiapy.calibrate import register, update_pointing +from aiapy.calibrate.util import get_pointing_table ############################################################################### # Performing multi-wavelength analysis on level 1 data can be problematic as @@ -42,10 +45,12 @@ ############################################################################### # The first step in this process is to update the metadata of the map to the # most recent pointing using the `aiapy.calibrate.update_pointing` function. -# This function queries the JSOC for the most recent pointing information, -# updates the metadata, and returns a `sunpy.map.Map` with updated metadata. +# One needs to get the pointing information from the JSOC using the +# `aiapy.calibrate.util.get_pointing_table` function. -aia_map_updated_pointing = update_pointing(aia_map) +# Make range wide enough to get closest 3-hour pointing +pointing_table = get_pointing_table("JSOC", start=aia_map.date - 12 * u.h, end=aia_map.date + 12 * u.h) +aia_map_updated_pointing = update_pointing(aia_map, pointing_table=pointing_table) ############################################################################### # If we take a look at the plate scale and rotation matrix of the map, we diff --git a/examples/skip_replace_hot_pixels.py b/examples/replace_hot_pixels.py similarity index 100% rename from examples/skip_replace_hot_pixels.py rename to examples/replace_hot_pixels.py diff --git a/examples/skip_download_specific_data.py b/examples/skip_download_specific_data.py index ca31855..6720968 100644 --- a/examples/skip_download_specific_data.py +++ b/examples/skip_download_specific_data.py @@ -58,9 +58,9 @@ level_1_maps = sunpy.map.Map(files) # We get the pointing table outside of the loop for the relevant time range. # Otherwise you're making a call to the JSOC every single time. -pointing_table = get_pointing_table(level_1_maps[0].date - 3 * u.h, level_1_maps[-1].date + 3 * u.h) +pointing_table = get_pointing_table("jsoc", start=level_1_maps[0].date - 3 * u.h, end=level_1_maps[-1].date + 3 * u.h) # The same applies for the correction table. -correction_table = get_correction_table(source=10) +correction_table = get_correction_table(source="jsoc") level_15_maps = [] for a_map in level_1_maps: diff --git a/examples/update_header_keywords.py b/examples/update_header_keywords.py index 0452f30..7d219e9 100644 --- a/examples/update_header_keywords.py +++ b/examples/update_header_keywords.py @@ -11,10 +11,13 @@ import matplotlib.pyplot as plt +import astropy.units as u + import sunpy.map import aiapy.data.sample as sample_data from aiapy.calibrate import fix_observer_location, update_pointing +from aiapy.calibrate.util import get_pointing_table ############################################################################### # An AIA FITS header contains various pieces of @@ -39,12 +42,13 @@ ############################################################################### # To update the pointing keywords, we can pass our `~sunpy.map.Map` to the -# `aiapy.calibrate.update_pointing` function. This function will query the -# JSOC, using `~sunpy`, for the most recent pointing information, update -# the metadata, and then return a new `~sunpy.map.Map` with this updated -# metadata. +# `aiapy.calibrate.update_pointing` function. +# One needs to get the pointing information from the JSOC using the +# `aiapy.calibrate.util.get_pointing_table` function first. -aia_map_updated_pointing = update_pointing(aia_map) +# Make range wide enough to get closest 3-hour pointing +pointing_table = get_pointing_table("JSOC", start=aia_map.date - 12 * u.h, end=aia_map.date + 12 * u.h) +aia_map_updated_pointing = update_pointing(aia_map, pointing_table=pointing_table) ############################################################################### # If we inspect the reference pixel and rotation matrix of the original map: diff --git a/pyproject.toml b/pyproject.toml index 53fd447..b92fbc5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,6 +69,7 @@ docs = [ "sunpy-sphinx-theme", "sunpy-sphinx-theme", ] +dev = ["aiapy[all,tests,docs]"] [project.urls] Homepage = "https://aia.lmsal.com/" diff --git a/tox.ini b/tox.ini index 0628bfe..29e2bf2 100644 --- a/tox.ini +++ b/tox.ini @@ -6,8 +6,9 @@ envlist = py{310,311,312,313} py313-devdeps py310-oldestdeps + cupy codestyle - build_docs{,-gallery} + build-docs{,-gallery} [testenv] pypi_filter = https://raw.githubusercontent.com/sunpy/sunpy/main/.test_package_pins.txt @@ -31,6 +32,7 @@ set_env = MPLBACKEND = agg devdeps: PIP_EXTRA_INDEX_URL = https://pypi.anaconda.org/astropy/simple https://pypi.anaconda.org/scientific-python-nightly-wheels/simple deps = + cupy: cupy-cuda12x # For packages which publish nightly wheels this will pull the latest nightly devdeps: astropy>=0.0.dev0 devdeps: sunpy>=0.0.dev0 @@ -70,7 +72,7 @@ commands = pre-commit install-hooks pre-commit run --color always --all-files --show-diff-on-failure -[testenv:build_docs{,-gallery}] +[testenv:build-docs{,-gallery}] description = invoke sphinx-build to build the HTML docs change_dir = docs From 5daff52c050c1cc504699ae6061cebc79336d222 Mon Sep 17 00:00:00 2001 From: Nabil Freij Date: Tue, 7 Jan 2025 10:06:12 -0800 Subject: [PATCH 07/11] Apply suggestions from code review Co-authored-by: Will Barnes --- changelog/346.breaking.rst | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/changelog/346.breaking.rst b/changelog/346.breaking.rst index 8a8a451..aad32e6 100644 --- a/changelog/346.breaking.rst +++ b/changelog/346.breaking.rst @@ -2,21 +2,21 @@ Due to the temporary shutdown of the JSOC, many of the functions within ``aiapy` In an effort to enable other sources of data to be provided to the broken functions, the following breaking changes have been made: -1. `aiapy.calibrate.correct_degradation` has removed "calibration_version" keyword and now the "correction_table" is now a required argument. +1. `aiapy.calibrate.correct_degradation`: The "calibration_version" keyword has been removed and "correction_table" is now a required argument. To get a correction table, one can use the `aiapy.calibrate.util.get_correction_table`. This allows one to select between, JSOC, SSW, or a custom correction table. For SSW, the correction table is just a version of the SSW correction table which goes from 3 to 10. If you want to use the JSOC, you can pass in "jsoc" as a string argument. -2. `aiapy.calibrate.estimate_error`, the "error_table" keyword is now a required argument. +2. `aiapy.calibrate.estimate_error`: "error_table" is now a required argument. To get the error table, one can use the `aiapy.calibrate.util.get_error_table`. There are only two options for the error table, 2 or 3. -3. `aiapy.response.channel.Channel.eve_correction`, the correction_table keyword is now a required argument. - As for `aiapy.calibrate.correct_degradation`, one can use the `aiapy.calibrate.util.get_correction_table` to get the correction table. +3. `aiapy.response.channel.Channel.eve_correction`: "correction_table" is now a required argument. + As in `aiapy.calibrate.correct_degradation`, one can use the `aiapy.calibrate.util.get_correction_table` to get the correction table. -4. `aiapy.calibrate.update_pointing`, the "pointing_table" keyword is now a required argument. +4. `aiapy.calibrate.update_pointing`: "pointing_table" is now a required argument. To get the pointing table, one can use `aiapy.calibrate.util.get_pointing_table`. This function now has a "source" keyword which can be used to select between JSOC and a copy of the pointing information dated from 11/20/2024 stored on LMSAL servers. -Not that many of the files from SSW are not updated with any frequency and provide worse results than using the data from the JSOC. +Note that many of the files from SSW are not updated with any frequency and provide worse results than using the data from the JSOC. From c6925703888ead01a4e974326c7f102733301989 Mon Sep 17 00:00:00 2001 From: Nabil Freij Date: Tue, 7 Jan 2025 11:14:09 -0800 Subject: [PATCH 08/11] Code review comments from Will --- aiapy/calibrate/spikes.py | 4 +- aiapy/calibrate/tests/test_meta.py | 2 +- aiapy/calibrate/tests/test_prep.py | 17 ++- aiapy/calibrate/tests/test_util.py | 19 +--- aiapy/calibrate/util.py | 168 ++++++++--------------------- aiapy/util/net.py | 4 +- aiapy/util/tests/test_net.py | 10 +- aiapy/util/util.py | 4 +- changelog/346.breaking.rst | 6 +- 9 files changed, 72 insertions(+), 162 deletions(-) diff --git a/aiapy/calibrate/spikes.py b/aiapy/calibrate/spikes.py index 979c3f3..d8fb582 100644 --- a/aiapy/calibrate/spikes.py +++ b/aiapy/calibrate/spikes.py @@ -15,7 +15,7 @@ from sunpy.map.sources.sdo import AIAMap from aiapy.util import AIApyUserWarning -from aiapy.util.net import get_data_from_jsoc +from aiapy.util.net import _get_data_from_jsoc __all__ = ["fetch_spikes", "respike"] @@ -146,7 +146,7 @@ def fetch_spikes(smap, *, as_coords=False): Original intensity values of the spikes """ series = "aia.lev1_uv_24s" if smap.wavelength in (1600, 1700, 4500) * u.angstrom else "aia.lev1_euv_12s" - file = get_data_from_jsoc(f'{series}[{smap.date}/12s][WAVELNTH={smap.meta["wavelnth"]}]', key=None, seg="spikes") + file = _get_data_from_jsoc(f'{series}[{smap.date}/12s][WAVELNTH={smap.meta["wavelnth"]}]', key=None, seg="spikes") _, spikes = fits.open(f'http://jsoc.stanford.edu{file["spikes"][0]}') # Loaded as floats, but they are actually integers spikes = spikes.data.astype(np.int32) diff --git a/aiapy/calibrate/tests/test_meta.py b/aiapy/calibrate/tests/test_meta.py index 954ab75..22fe1e9 100644 --- a/aiapy/calibrate/tests/test_meta.py +++ b/aiapy/calibrate/tests/test_meta.py @@ -54,7 +54,7 @@ def test_fix_pointing(aia_171_map, pointing_table) -> None: aia_171_map.meta.pop(k) aia_map_updated = update_pointing( aia_171_map, - pointing_table=get_pointing_table("lmsal"), + pointing_table=pointing_table, ) # FIXME: how do we check these values are accurate? assert all(k in aia_map_updated.meta for k in keys) diff --git a/aiapy/calibrate/tests/test_prep.py b/aiapy/calibrate/tests/test_prep.py index 7a2949b..14396a3 100644 --- a/aiapy/calibrate/tests/test_prep.py +++ b/aiapy/calibrate/tests/test_prep.py @@ -103,7 +103,7 @@ def test_register_level_15(lvl_15_map) -> None: ("source"), [ pytest.param("jsoc", marks=pytest.mark.remote_data), - pytest.param(8, marks=pytest.mark.remote_data), + pytest.param("SsW", marks=pytest.mark.remote_data), get_test_filepath("aia_V8_20171210_050627_response_table.txt"), ], ) @@ -129,18 +129,13 @@ def test_correct_degradation(aia_171_map, source) -> None: ("source", "time_correction_truth"), [ pytest.param( - 10, + "SSW", 0.9031773242843387 * u.dimensionless_unscaled, marks=pytest.mark.remote_data, ), pytest.param( - 9, - 0.8658650561969473 * u.dimensionless_unscaled, - marks=pytest.mark.remote_data, - ), - pytest.param( - 8, - 0.7667012041798814 * u.dimensionless_unscaled, + "JSOC", + 0.86288462 * u.dimensionless_unscaled, marks=pytest.mark.remote_data, ), ( @@ -214,7 +209,7 @@ def test_degradation_all_wavelengths(wavelength, result) -> None: time_correction = degradation( wavelength * u.angstrom, obstime, - correction_table=get_correction_table(10), + correction_table=get_correction_table("SSW"), ) assert u.allclose(time_correction, result, atol=1e-3) @@ -227,7 +222,7 @@ def test_degradation_4500_missing() -> None: ValueError, match="Correction table does not contain calibration for 4500.0 Angstrom. Max version is 3.", ): - degradation(4500 * u.angstrom, obstime, correction_table=get_correction_table(10)) + degradation(4500 * u.angstrom, obstime, correction_table=get_correction_table("SSW")) @pytest.mark.remote_data diff --git a/aiapy/calibrate/tests/test_util.py b/aiapy/calibrate/tests/test_util.py index 159e8fc..574daeb 100644 --- a/aiapy/calibrate/tests/test_util.py +++ b/aiapy/calibrate/tests/test_util.py @@ -22,14 +22,8 @@ @pytest.mark.parametrize( "source", [ - pytest.param("jsoc", marks=pytest.mark.remote_data), - pytest.param(3, marks=pytest.mark.remote_data), - pytest.param(4, marks=pytest.mark.remote_data), - pytest.param(6, marks=pytest.mark.remote_data), - pytest.param(7, marks=pytest.mark.remote_data), - pytest.param(8, marks=pytest.mark.remote_data), - pytest.param(9, marks=pytest.mark.remote_data), - pytest.param(10, marks=pytest.mark.remote_data), + pytest.param("JSoC", marks=pytest.mark.remote_data), # To check the lower case comparison is working + pytest.param("SsW", marks=pytest.mark.remote_data), # To check the lower case comparison is workings get_test_filepath("aia_V8_20171210_050627_response_table.txt"), ], ) @@ -75,9 +69,7 @@ def test_correction_table_selection(wavelength) -> None: def test_invalid_correction_table_input() -> None: with pytest.raises( ValueError, - match=re.escape( - "correction_table must be a file path (pathlib.Path), 'jsoc' or one of 3, 4, 6, 7, 8, 9, 10. Not -1" - ), + match=re.escape("correction_table must be a file path (pathlib.Path), 'JSOC' or 'SSW'. Not -1"), ): get_correction_table(source=-1) @@ -127,8 +119,7 @@ def test_pointing_table_unavailable() -> None: @pytest.mark.parametrize( "error_table", [ - pytest.param(2, marks=pytest.mark.remote_data), - pytest.param(3, marks=pytest.mark.remote_data), + pytest.param("SSW", marks=pytest.mark.remote_data), get_test_filepath("aia_V3_error_table.txt"), ], ) @@ -139,5 +130,5 @@ def test_error_table(error_table) -> None: def test_invalid_error_table_input() -> None: - with pytest.raises(TypeError, match="source must be a file path, or 2 or 3, not -1"): + with pytest.raises(TypeError, match="source must be a pathlib.Path, or 'SSW', not -1"): get_error_table(-1) diff --git a/aiapy/calibrate/util.py b/aiapy/calibrate/util.py index 7aad333..f44e9c7 100644 --- a/aiapy/calibrate/util.py +++ b/aiapy/calibrate/util.py @@ -20,7 +20,7 @@ from aiapy import _SSW_MIRRORS from aiapy.data._manager import manager from aiapy.util.decorators import validate_channel -from aiapy.util.net import get_data_from_jsoc +from aiapy.util.net import _get_data_from_jsoc __all__ = [ "get_correction_table", @@ -32,95 +32,26 @@ _AIA_ERROR_FILE = "sdo/aia/response/aia_V{}_error_table.txt" # URLs and SHA-256 hashes for each version of the error tables _URL_HASH_ERROR_TABLE = { - 2: ( - [urljoin(mirror, _AIA_ERROR_FILE.format(2)) for mirror in _SSW_MIRRORS], - "ac97ccc48057809723c27e3ef290c7d78ee35791d9054b2188baecfb5c290d0a", - ), 3: ( [urljoin(mirror, _AIA_ERROR_FILE.format(3)) for mirror in _SSW_MIRRORS], "66ff034923bb0fd1ad20e8f30c7d909e1a80745063957dd6010f81331acaf894", - ), + ) } _URL_HASH_POINTING_TABLE = ( "https://aia.lmsal.com/public/master_aia_pointing3h.csv", "a2c80fa0ea3453c62c91f51df045ae04b771d5cbb51c6495ed56de0da2a5482e", ) -_URL_HASH_RESPONSE_TABLE = { +_URL_HASH_CORRECTION_TABLE = { 10: ( [urljoin(mirror, "sdo/aia/response/aia_V10_20201119_190000_response_table.txt") for mirror in _SSW_MIRRORS], "0a3f2db39d05c44185f6fdeec928089fb55d1ce1e0a805145050c6356cbc6e98", - ), - 9: ( - [urljoin(mirror, "sdo/aia/response/aia_V9_20200706_215452_response_table.txt") for mirror in _SSW_MIRRORS], - "f24b384cba9935ae2e8fd3c0644312720cb6add95c49ba46f1961ae4cf0865f9", - ), - 8: ( - [urljoin(mirror, "sdo/aia/response/aia_V8_20171210_050627_response_table.txt") for mirror in _SSW_MIRRORS], - "0e8bc6af5a69f80ca9d4fc2a27854681b76574d59eb81d7201b7f618081f0fdd", - ), - 7: ( - [urljoin(mirror, "sdo/aia/response/aia_V7_20171129_195626_response_table.txt") for mirror in _SSW_MIRRORS], - "ac2171d549bd6cc6c37e13e505eef1bf0c89fc49bffd037e4ac64f0b895063ac", - ), - 6: ( - [urljoin(mirror, "sdo/aia/response/aia_V6_20141027_230030_response_table.txt") for mirror in _SSW_MIRRORS], - "11c148f447d4538db8fd247f74c26b4ae673355e2536f63eb48f9a267e58c7c6", - ), - 4: ( - [urljoin(mirror, "sdo/aia/response/aia_V4_20130109_204835_response_table.txt") for mirror in _SSW_MIRRORS], - "7e73f4effa9a8dc55f7b4993a8d181419ef555bf295c4704703ca84d7a0fc3c1", - ), - 3: ( - [urljoin(mirror, "sdo/aia/response/aia_V3_20120926_201221_response_table.txt") for mirror in _SSW_MIRRORS], - "0a5d2c2ed1cda18bb9fbdbd51fbf3374e042d20145150632ac95350fc99de68b", - ), - 2: ( - [urljoin(mirror, "sdo/aia/response/aia_V2_20111129_000000_response_table.txt") for mirror in _SSW_MIRRORS], - "d55ccd6cb3cb4bd1c688f8663f942f8a872c918a2504e5e474aa97dff45b62c9", - ), + ) } -def _fetch_response_table(version: int): - # Until the delayed feature from sunpy (v6.1) is out, this function - # will need to be like this. - if version not in _URL_HASH_RESPONSE_TABLE: - msg = f"Invalid response table version: {version}" - raise ValueError(msg) - - @manager.require("response_table_v10", *_URL_HASH_RESPONSE_TABLE[10]) - def fetch_response_table_v10(): - return manager.get("response_table_v10") - - @manager.require("response_table_v9", *_URL_HASH_RESPONSE_TABLE[9]) - def fetch_response_table_v9(): - return manager.get("response_table_v9") - - @manager.require("response_table_v8", *_URL_HASH_RESPONSE_TABLE[8]) - def fetch_response_table_v8(): - return manager.get("response_table_v8") - - @manager.require("response_table_v7", *_URL_HASH_RESPONSE_TABLE[7]) - def fetch_response_table_v7(): - return manager.get("response_table_v7") - - @manager.require("response_table_v6", *_URL_HASH_RESPONSE_TABLE[6]) - def fetch_response_table_v6(): - return manager.get("response_table_v6") - - @manager.require("response_table_v4", *_URL_HASH_RESPONSE_TABLE[4]) - def fetch_response_table_v4(): - return manager.get("response_table_v4") - - @manager.require("response_table_v3", *_URL_HASH_RESPONSE_TABLE[3]) - def fetch_response_table_v3(): - return manager.get("response_table_v3") - - @manager.require("response_table_v2", *_URL_HASH_RESPONSE_TABLE[2]) - def fetch_response_table_v2(): - return manager.get("response_table_v2") - - return locals()[f"fetch_response_table_v{version}"]() +@manager.require("correction_table_v10", *_URL_HASH_CORRECTION_TABLE[10]) +def _fetch_correction_table_v10(): + return manager.get("correction_table_v10") def get_correction_table(source): @@ -137,11 +68,10 @@ def get_correction_table(source): Parameters ---------- - source: pathlib.Path, str or int + source: pathlib.Path, str The source of the correction table. If it is a `pathlib.Path`, it must be a file. A string file path will error as an invalid source. - If a string, it must be "jsoc" which will fetch the most recent version from the JSOC, - otherwise an integer: 3, 4, 6, 7, 8, 9, 10 to use fixed version files from SSW. + If source is a string, it must either be "JSOC" which will fetch the most recent version from the JSOC or "SSW" which will fetch the most recent version (V10) from SSW. Returns ------- @@ -154,19 +84,17 @@ def get_correction_table(source): """ if isinstance(source, pathlib.Path): table = QTable(astropy.io.ascii.read(source)) - elif source in _URL_HASH_RESPONSE_TABLE: - table = QTable(astropy.io.ascii.read(_fetch_response_table(source))) + elif isinstance(source, str) and source.lower() == "ssw": + table = QTable(astropy.io.ascii.read(_fetch_correction_table_v10())) elif isinstance(source, str) and source.lower() == "jsoc": # NOTE: the [!1=1!] disables the drms PrimeKey logic and enables # the query to find records that are ordinarily considered # identical because the PrimeKeys for this series are WAVE_STR # and T_START. Without the !1=1! the query only returns the # latest record for each unique combination of those keywords. - table = QTable.from_pandas(get_data_from_jsoc(query="aia.response[][!1=1!]", key="**ALL**")) + table = QTable.from_pandas(_get_data_from_jsoc(query="aia.response[][!1=1!]", key="**ALL**")) else: - msg = ( - f"correction_table must be a file path (pathlib.Path), 'jsoc' or one of 3, 4, 6, 7, 8, 9, 10. Not {source}" - ) + msg = f"correction_table must be a file path (pathlib.Path), 'JSOC' or 'SSW'. Not {source}" raise ValueError(msg) selected_cols = [ "DATE", @@ -238,7 +166,7 @@ def _select_epoch_from_correction_table(channel: u.angstrom, obstime, correction @manager.require("pointing_table", *_URL_HASH_POINTING_TABLE) -def fetch_pointing_table(): +def _fetch_pointing_table(): return manager.get("pointing_table") @@ -254,24 +182,33 @@ def get_pointing_table(source, *, start=None, end=None): The 3-hourly MPT entries are computed from limb fits of images with ``T_OBS`` between ``T_START`` and ``T_STOP``. - .. note:: A MPT entry covers the interval ``[T_START:T_STOP)``; - that is, the interval includes ``T_START`` and excludes - ``T_STOP``. + .. note:: + + A MPT entry covers the interval ``[T_START:T_STOP)``; + that is, the interval includes ``T_START`` and excludes + ``T_STOP``. + + .. note:: - .. note:: While it is generally true that ``TSTOP = T_START + 3 hours``, - there are edge cases where ``T_STOP`` is more than 3 hours - after ``T_START`` because of a calibration, an eclipse, - or other reasons, but the fits are still calculated based - on images from ``T_START`` to ``T_START + 3 hours``. - Pointing is not stable during these periods, so the question - of which MPT entry to use is not relevant. + While it is generally true that ``TSTOP = T_START + 3 hours``, + there are edge cases where ``T_STOP`` is more than 3 hours + after ``T_START`` because of a calibration, an eclipse, + or other reasons, but the fits are still calculated based + on images from ``T_START`` to ``T_START + 3 hours``. + Pointing is not stable during these periods, so the question + of which MPT entry to use is not relevant. + + .. note:: + + The LMSAL pointing table is a static copy of the JSOC table dated from 11/20/2024. + It was designed as a stop-gap measure while the JSOC recovered from its recent + water damage. Parameters ---------- source : str Name of the source from which to retrieve the pointing table. Must be one of ``"jsoc"`` or ``"lmsal"``. - Note that the LMSAL pointing table is not updated frequently. start : `~astropy.time.Time`, optional Start time of the interval. Only required if ``source`` is ``"jsoc"``. end : `~astropy.time.Time`, optional @@ -290,10 +227,10 @@ def get_pointing_table(source, *, start=None, end=None): msg = "start and end must be provided if source is 'jsoc'" raise ValueError(msg) table = QTable.from_pandas( - get_data_from_jsoc(query=f"aia.master_pointing3h[{start.isot}Z-{end.isot}Z]", key="**ALL**") + _get_data_from_jsoc(query=f"aia.master_pointing3h[{start.isot}Z-{end.isot}Z]", key="**ALL**") ) elif source.lower() == "lmsal": - table = QTable(astropy_ascii.read(fetch_pointing_table())) + table = QTable(astropy_ascii.read(_fetch_pointing_table())) else: msg = f"Invalid source: {source}, must be one of 'jsoc' or 'lmsal'" raise ValueError(msg) @@ -313,36 +250,23 @@ def get_pointing_table(source, *, start=None, end=None): return table -def _fetch_error_table(version: int): - # Until the delayed feature from sunpy (v6.1) is out, this function - # will need to be like this. - @manager.require("error_table_v2", *_URL_HASH_ERROR_TABLE[2]) - def fetch_error_table_v2(): - return manager.get("error_table_v2") - - @manager.require("error_table_v3", *_URL_HASH_ERROR_TABLE[3]) - def fetch_error_table_v3(): - return manager.get("error_table_v3") - - if version == 2: - return fetch_error_table_v2() - if version == 3: - return fetch_error_table_v3() - msg = f"Invalid error table version: {version}, must be 2 or 3" - raise ValueError(msg) +@manager.require("error_table_v3", *_URL_HASH_ERROR_TABLE[3]) +def _fetch_error_table_v3(): + return manager.get("error_table_v3") -def get_error_table(source) -> QTable: +def get_error_table(source="SSW") -> QTable: """ Fetches the error table from a SSW mirror or uses a local file if one is provided. Parameters ---------- - source : pathlib.Path, int + source : pathlib.Path, str, optional If input is a pathlib.Path, it is assumed to be a file path to a local error table. If a path is provided as a string, it will error as an invalid source. - If the input is an int, it is assumed to be a version of the error table (2, 3). + Otherwise, the input is allowed to be "SSW" (the default) which will + fetch the most recent version (V3) from SSW. Returns ------- @@ -356,10 +280,10 @@ def get_error_table(source) -> QTable: """ if isinstance(source, pathlib.Path): error_table = QTable(astropy.io.ascii.read(source)) - elif source in [2, 3]: - error_table = QTable(astropy.io.ascii.read(_fetch_error_table(source))) + elif isinstance(source, str) and source.lower() == "ssw": + error_table = QTable(astropy.io.ascii.read(_fetch_error_table_v3())) else: - msg = f"source must be a file path, or 2 or 3, not {source}" + msg = f"source must be a pathlib.Path, or 'SSW', not {source}" raise TypeError(msg) error_table["DATE"] = Time(error_table["DATE"], scale="utc") error_table["T_START"] = Time(error_table["T_START"], scale="utc") diff --git a/aiapy/util/net.py b/aiapy/util/net.py index 3b579b0..d055781 100644 --- a/aiapy/util/net.py +++ b/aiapy/util/net.py @@ -4,10 +4,10 @@ import drms -__all__ = ["get_data_from_jsoc"] +__all__ = ["_get_data_from_jsoc"] -def get_data_from_jsoc(query, *, key, seg=None): +def _get_data_from_jsoc(query, *, key, seg=None): """ A simple wrapper around `~drms.Client.query` that raises a more informative error message. diff --git a/aiapy/util/tests/test_net.py b/aiapy/util/tests/test_net.py index 1b5b576..04cf3f6 100644 --- a/aiapy/util/tests/test_net.py +++ b/aiapy/util/tests/test_net.py @@ -1,13 +1,13 @@ import pytest -from aiapy.util.net import get_data_from_jsoc +from aiapy.util.net import _get_data_from_jsoc @pytest.mark.remote_data -def test_get_data_from_jsoc(): - assert get_data_from_jsoc("aia.master_pointing3h[2010-05-13T00:00:00Z]", key="**ALL**") is not None +def test__get_data_from_jsoc(): + assert _get_data_from_jsoc("aia.master_pointing3h[2010-05-13T00:00:00Z]", key="**ALL**") is not None -def test_get_data_from_jsoc_error(): +def test__get_data_from_jsoc_error(): with pytest.raises(OSError, match="Unable to query the JSOC"): - get_data_from_jsoc("abc", key="def") + _get_data_from_jsoc("abc", key="def") diff --git a/aiapy/util/util.py b/aiapy/util/util.py index 2ba4b88..e356b3a 100644 --- a/aiapy/util/util.py +++ b/aiapy/util/util.py @@ -11,7 +11,7 @@ from sunpy.time import parse_time from aiapy.util.decorators import validate_channel -from aiapy.util.net import get_data_from_jsoc +from aiapy.util.net import _get_data_from_jsoc __all__ = ["sdo_location", "telescope_number"] @@ -36,7 +36,7 @@ def sdo_location(time): """ t = parse_time(time) # Query for +/- 3 seconds around the given time - keys = get_data_from_jsoc(query=f"aia.lev1[{(t - 3*u.s).utc.isot}/6s]", key="T_OBS, HAEX_OBS, HAEY_OBS, HAEZ_OBS") + keys = _get_data_from_jsoc(query=f"aia.lev1[{(t - 3*u.s).utc.isot}/6s]", key="T_OBS, HAEX_OBS, HAEY_OBS, HAEZ_OBS") if keys is None or len(keys) == 0: msg = f"No JSOC records near this time: {t}" raise ValueError(msg) diff --git a/changelog/346.breaking.rst b/changelog/346.breaking.rst index aad32e6..9c9b0fd 100644 --- a/changelog/346.breaking.rst +++ b/changelog/346.breaking.rst @@ -5,18 +5,18 @@ In an effort to enable other sources of data to be provided to the broken functi 1. `aiapy.calibrate.correct_degradation`: The "calibration_version" keyword has been removed and "correction_table" is now a required argument. To get a correction table, one can use the `aiapy.calibrate.util.get_correction_table`. This allows one to select between, JSOC, SSW, or a custom correction table. - For SSW, the correction table is just a version of the SSW correction table which goes from 3 to 10. + For SSW, the correction table will be the latest version of the SSW correction table (V10). If you want to use the JSOC, you can pass in "jsoc" as a string argument. 2. `aiapy.calibrate.estimate_error`: "error_table" is now a required argument. To get the error table, one can use the `aiapy.calibrate.util.get_error_table`. - There are only two options for the error table, 2 or 3. + By default, this function will fetch the most recent version of the error table (V3) from SSW. 3. `aiapy.response.channel.Channel.eve_correction`: "correction_table" is now a required argument. As in `aiapy.calibrate.correct_degradation`, one can use the `aiapy.calibrate.util.get_correction_table` to get the correction table. 4. `aiapy.calibrate.update_pointing`: "pointing_table" is now a required argument. To get the pointing table, one can use `aiapy.calibrate.util.get_pointing_table`. - This function now has a "source" keyword which can be used to select between JSOC and a copy of the pointing information dated from 11/20/2024 stored on LMSAL servers. + This function now has a "source" keyword which can be used to select between "JSOC" (plus a start and end time) and "LMSAL" to get a copy of the pointing information dated from 11/20/2024 stored on LMSAL servers. Note that many of the files from SSW are not updated with any frequency and provide worse results than using the data from the JSOC. From 9f56c8137ee0ab0fe345b6032e8ff45b181a563d Mon Sep 17 00:00:00 2001 From: Nabil Freij Date: Tue, 7 Jan 2025 11:27:15 -0800 Subject: [PATCH 09/11] Remove fix_location --- aiapy/calibrate/meta.py | 43 +--------------------------- aiapy/calibrate/tests/test_meta.py | 10 +------ changelog/346.breaking.1.rst | 1 + examples/update_header_keywords.py | 46 ++++++------------------------ 4 files changed, 11 insertions(+), 89 deletions(-) create mode 100644 changelog/346.breaking.1.rst diff --git a/aiapy/calibrate/meta.py b/aiapy/calibrate/meta.py index 7371873..5efa5f0 100644 --- a/aiapy/calibrate/meta.py +++ b/aiapy/calibrate/meta.py @@ -8,53 +8,12 @@ import numpy as np import astropy.units as u -from astropy.coordinates import CartesianRepresentation, HeliocentricMeanEcliptic, SkyCoord from sunpy.map import contains_full_disk from aiapy.util.exceptions import AIApyUserWarning -__all__ = ["fix_observer_location", "update_pointing"] - - -def fix_observer_location(smap): - """ - Fix inaccurate ``HGS_LON`` and ``HGS_LAT`` FITS keywords. - - The heliographic Stonyhurst latitude and longitude locations in the - AIA FITS headers are incorrect. This function fixes the values of these - keywords using the heliocentric aries ecliptic keywords, ``HAEX_OBS, - HAEY_OBS, HAEZ_OBS``. - - .. note:: - - `~sunpy.map.sources.AIAMap` already accounts for the inaccurate - HGS keywords by using the HAE keywords to construct the - derived observer location. - - Parameters - ---------- - smap : `~sunpy.map.sources.AIAMap` - Input map. - - Returns - ------- - `~sunpy.map.sources.AIAMap` - """ - # Create observer coordinate from HAE coordinates (reason?) - coord = SkyCoord( - x=smap.meta["haex_obs"] * u.m, - y=smap.meta["haey_obs"] * u.m, - z=smap.meta["haez_obs"] * u.m, - representation_type=CartesianRepresentation, - frame=HeliocentricMeanEcliptic, - obstime=smap.reference_date, - ).heliographic_stonyhurst - new_meta = copy.deepcopy(smap.meta) - new_meta["hgln_obs"] = coord.lon.to(u.degree).value - new_meta["hglt_obs"] = coord.lat.to(u.degree).value - new_meta["dsun_obs"] = coord.radius.to(u.m).value - return smap._new_instance(smap.data, new_meta, plot_settings=smap.plot_settings, mask=smap.mask) +__all__ = ["update_pointing"] def update_pointing(smap, *, pointing_table): diff --git a/aiapy/calibrate/tests/test_meta.py b/aiapy/calibrate/tests/test_meta.py index 22fe1e9..faeaed9 100644 --- a/aiapy/calibrate/tests/test_meta.py +++ b/aiapy/calibrate/tests/test_meta.py @@ -6,19 +6,11 @@ from astropy.table import QTable from astropy.time import Time, TimeDelta -from aiapy.calibrate import fix_observer_location, update_pointing +from aiapy.calibrate import update_pointing from aiapy.calibrate.util import get_pointing_table from aiapy.util.exceptions import AIApyUserWarning -def test_fix_observer_location(aia_171_map) -> None: - smap_fixed = fix_observer_location(aia_171_map) - # NOTE: AIAMap already fixes the .observer_coordinate property with HAE - assert smap_fixed.meta["hgln_obs"] == smap_fixed.observer_coordinate.lon.value - assert smap_fixed.meta["hglt_obs"] == smap_fixed.observer_coordinate.lat.value - assert smap_fixed.meta["dsun_obs"] == smap_fixed.observer_coordinate.radius.value - - @pytest.fixture def pointing_table(): return get_pointing_table("lmsal") diff --git a/changelog/346.breaking.1.rst b/changelog/346.breaking.1.rst new file mode 100644 index 0000000..a5d772e --- /dev/null +++ b/changelog/346.breaking.1.rst @@ -0,0 +1 @@ +Removed ``fix_observer_location`` as it is no longer needed. diff --git a/examples/update_header_keywords.py b/examples/update_header_keywords.py index 7d219e9..f013492 100644 --- a/examples/update_header_keywords.py +++ b/examples/update_header_keywords.py @@ -1,12 +1,11 @@ """ -========================================================== -Updating pointing and observer keywords in the FITS header -========================================================== +============================================= +Updating pointing keywords in the FITS header +============================================= This example demonstrates how to update the metadata in an AIA FITS file to ensure that it has the most accurate -information regarding the spacecraft pointing and observer -position. +information regarding the spacecraft pointing. """ import matplotlib.pyplot as plt @@ -16,7 +15,7 @@ import sunpy.map import aiapy.data.sample as sample_data -from aiapy.calibrate import fix_observer_location, update_pointing +from aiapy.calibrate import update_pointing from aiapy.calibrate.util import get_pointing_table ############################################################################### @@ -67,45 +66,16 @@ # and ``CROTA2``, have been updated. # # Similarly, the Heliographic Stonyhurst (HGS) coordinates of the observer -# location in the header are inaccurate. If we check the HGS longitude keyword -# in the header, we find that it is 0 degrees which is not the HGS longitude -# coordinate of SDO. +# location in the header are correct. print(aia_map_updated_pointing.meta["hgln_obs"]) print(aia_map_updated_pointing.meta["hglt_obs"]) ############################################################################### -# To update the HGS observer coordinates, we can use the -# `aiapy.calibrate.fix_observer_location` function. This function reads the -# correct observer location from Heliocentric Aries Ecliptic (HAE) coordinates -# in the header, converts them to HGS, and replaces the inaccurate HGS -# keywords. - -aia_map_observer_fixed = fix_observer_location(aia_map_updated_pointing) - -############################################################################### -# Looking again at the HGS longitude and latitude keywords, we can see that -# they have been updated. -print(aia_map_observer_fixed.meta["hgln_obs"]) -print(aia_map_observer_fixed.meta["hglt_obs"]) - -############################################################################### -# Note that in `~sunpy.map.sources.AIAMap`, the `~sunpy.map.GenericMap.observer_coordinate` -# attribute is already derived from the HAE coordinates such that it is not -# strictly necessary to apply `aiapy.calibrate.fix_observer_location`. For -# example, the unfixed `~sunpy.map.Map` will still have an accurate derived -# observer position - -print(aia_map_updated_pointing.observer_coordinate) - -############################################################################### -# However, we suggest that users apply this fix such that the information -# stored in `~sunpy.map.GenericMap.meta` is accurate and consistent. -# # Finally, plot the fixed map. fig = plt.figure() -ax = fig.add_subplot(projection=aia_map_observer_fixed) -aia_map_observer_fixed.plot(axes=ax) +ax = fig.add_subplot(projection=aia_map_updated_pointing) +aia_map_updated_pointing.plot(axes=ax) plt.show() From 278ee4cf72cbddbeb2ca00c52fd8e7ab184bf7c5 Mon Sep 17 00:00:00 2001 From: Nabil Freij Date: Wed, 8 Jan 2025 11:40:57 -0800 Subject: [PATCH 10/11] Final review comments --- aiapy/calibrate/tests/test_util.py | 36 +++++------ aiapy/calibrate/util.py | 14 ++--- docs/preparing_data.rst | 1 - examples/prepping_level_1_data.py | 2 +- examples/skip_download_specific_data.py | 4 +- examples/update_header_keywords.py | 81 ------------------------- 6 files changed, 29 insertions(+), 109 deletions(-) delete mode 100644 examples/update_header_keywords.py diff --git a/aiapy/calibrate/tests/test_util.py b/aiapy/calibrate/tests/test_util.py index 574daeb..81cbc50 100644 --- a/aiapy/calibrate/tests/test_util.py +++ b/aiapy/calibrate/tests/test_util.py @@ -2,9 +2,9 @@ import pytest -import astropy.table -import astropy.time import astropy.units as u +from astropy.table import QTable +from astropy.time import Time from aiapy.calibrate.util import ( _select_epoch_from_correction_table, @@ -15,7 +15,7 @@ from aiapy.tests.data import get_test_filepath # These are not fixtures so that they can be easily used in the parametrize mark -obstime = astropy.time.Time("2015-01-01T00:00:00", scale="utc") +obstime = Time("2015-01-01T00:00:00", scale="utc") correction_table_local = get_correction_table(get_test_filepath("aia_V8_20171210_050627_response_table.txt")) @@ -29,7 +29,7 @@ ) def test_correction_table(source) -> None: table = get_correction_table(source=source) - assert isinstance(table, astropy.table.QTable) + assert isinstance(table, QTable) expected_columns = [ "VER_NUM", "WAVE_STR", @@ -42,14 +42,14 @@ def test_correction_table(source) -> None: "EFF_WVLN", ] assert all(cn in table.colnames for cn in expected_columns) - assert isinstance(table["T_START"], astropy.time.Time) - assert isinstance(table["T_STOP"], astropy.time.Time) + assert isinstance(table["T_START"], Time) + assert isinstance(table["T_STOP"], Time) @pytest.mark.parametrize("wavelength", [94 * u.angstrom, 1600 * u.angstrom]) def test_correction_table_selection(wavelength) -> None: table = _select_epoch_from_correction_table(wavelength, obstime, correction_table_local) - assert isinstance(table, astropy.table.QTable) + assert isinstance(table, QTable) expected_columns = [ "VER_NUM", "WAVE_STR", @@ -62,8 +62,8 @@ def test_correction_table_selection(wavelength) -> None: "EFF_WVLN", ] assert all(cn in table.colnames for cn in expected_columns) - assert isinstance(table["T_START"], astropy.time.Time) - assert isinstance(table["T_STOP"], astropy.time.Time) + assert isinstance(table["T_START"], Time) + assert isinstance(table["T_STOP"], Time) def test_invalid_correction_table_input() -> None: @@ -80,7 +80,7 @@ def test_invalid_wavelength_raises_exception() -> None: def test_obstime_out_of_range() -> None: - obstime_out_of_range = astropy.time.Time("2000-01-01T12:00:00", scale="utc") + obstime_out_of_range = Time("2000-01-01T12:00:00", scale="utc") with pytest.raises(ValueError, match=f"No valid calibration epoch for {obstime_out_of_range}"): _select_epoch_from_correction_table(94 * u.angstrom, obstime_out_of_range, correction_table_local) @@ -95,14 +95,14 @@ def test_pointing_table() -> None: f"A_{c}_IMSCALE", f"A_{c}_IMSCALE", ] - t = astropy.time.Time("2011-01-01T00:00:00", scale="utc") + t = Time("2011-01-01T00:00:00", scale="utc") table_lmsal = get_pointing_table("lmsal") - table_jsoc = get_pointing_table("jsoc", start=t - 3 * u.h, end=t + 3 * u.h) + table_jsoc = get_pointing_table("jsoc", time_range=(t - 3 * u.h, t + 3 * u.h)) for table in [table_lmsal, table_jsoc]: - assert isinstance(table, astropy.table.QTable) + assert isinstance(table, QTable) assert all(cn in table.colnames for cn in expected_columns) - assert isinstance(table["T_START"], astropy.time.Time) - assert isinstance(table["T_STOP"], astropy.time.Time) + assert isinstance(table["T_START"], Time) + assert isinstance(table["T_STOP"], Time) # Ensure that none of the pointing parameters are masked columns for c in expected_columns[2:]: assert not hasattr(table[c], "mask") @@ -111,9 +111,9 @@ def test_pointing_table() -> None: @pytest.mark.remote_data def test_pointing_table_unavailable() -> None: # Check that missing pointing data raises a nice error - t = astropy.time.Time("1990-01-01") + t = Time("1990-01-01") with pytest.raises(RuntimeError, match="No data found for this query"): - get_pointing_table("jsoc", start=t - 3 * u.h, end=t + 3 * u.h) + get_pointing_table("jsoc", time_range=Time([t - 3 * u.h, t + 3 * u.h])) @pytest.mark.parametrize( @@ -125,7 +125,7 @@ def test_pointing_table_unavailable() -> None: ) def test_error_table(error_table) -> None: table = get_error_table(error_table) - assert isinstance(table, astropy.table.QTable) + assert isinstance(table, QTable) assert len(table) == 10 diff --git a/aiapy/calibrate/util.py b/aiapy/calibrate/util.py index f44e9c7..0d98223 100644 --- a/aiapy/calibrate/util.py +++ b/aiapy/calibrate/util.py @@ -170,7 +170,7 @@ def _fetch_pointing_table(): return manager.get("pointing_table") -def get_pointing_table(source, *, start=None, end=None): +def get_pointing_table(source, *, time_range=None): """ Retrieve 3-hourly master pointing table from the given source. @@ -209,10 +209,9 @@ def get_pointing_table(source, *, start=None, end=None): source : str Name of the source from which to retrieve the pointing table. Must be one of ``"jsoc"`` or ``"lmsal"``. - start : `~astropy.time.Time`, optional - Start time of the interval. Only required if ``source`` is ``"jsoc"``. - end : `~astropy.time.Time`, optional - End time of the interval. Only required if ``source`` is ``"jsoc"``. + time_range : `~astropy.time.Time`, optional + Time range for which to retrieve the pointing table. + You can pass in a `~astropy.time.Time` object or a tuple of start and end times. Returns ------- @@ -223,9 +222,10 @@ def get_pointing_table(source, *, start=None, end=None): aiapy.calibrate.update_pointing """ if source.lower() == "jsoc": - if start is None or end is None: - msg = "start and end must be provided if source is 'jsoc'" + if time_range is None: + msg = "time_range must be provided if the source is 'jsoc'" raise ValueError(msg) + start, end = time_range table = QTable.from_pandas( _get_data_from_jsoc(query=f"aia.master_pointing3h[{start.isot}Z-{end.isot}Z]", key="**ALL**") ) diff --git a/docs/preparing_data.rst b/docs/preparing_data.rst index b3125d5..2133b19 100644 --- a/docs/preparing_data.rst +++ b/docs/preparing_data.rst @@ -31,6 +31,5 @@ If you want to do any additional data processing steps (e.g., PSF deconvolution) * The PSF functions are defined on the level 1 pixel grid so PSF deconvolution **MUST** be done on the level 1 data products (i.e., before image registration). This is described in the PSF gallery example :ref:`sphx_glr_generated_gallery_skip_psf_deconvolution.py`. * The pointing update should be done prior to image registration as the updated keywords, namely ``CRPIX1`` and ``CRPIX2``, are used in the image registration step. - More details can be found in this gallery example :ref:`sphx_glr_generated_gallery_update_header_keywords.py`. * The exposure time normalization and degradation correction (`aiapy.calibrate.correct_degradation`) operations are just scalar multiplication and are thus linear such that their ordering is inconsequential. * Exposure time normalization can be performed by simply dividing a map by the exposure time property, ``my_map / my_map.exposure_time``. diff --git a/examples/prepping_level_1_data.py b/examples/prepping_level_1_data.py index cb66909..fad759c 100644 --- a/examples/prepping_level_1_data.py +++ b/examples/prepping_level_1_data.py @@ -49,7 +49,7 @@ # `aiapy.calibrate.util.get_pointing_table` function. # Make range wide enough to get closest 3-hour pointing -pointing_table = get_pointing_table("JSOC", start=aia_map.date - 12 * u.h, end=aia_map.date + 12 * u.h) +pointing_table = get_pointing_table("JSOC", time_range=(aia_map.date - 12 * u.h, aia_map.date + 12 * u.h)) aia_map_updated_pointing = update_pointing(aia_map, pointing_table=pointing_table) ############################################################################### diff --git a/examples/skip_download_specific_data.py b/examples/skip_download_specific_data.py index 6720968..6914275 100644 --- a/examples/skip_download_specific_data.py +++ b/examples/skip_download_specific_data.py @@ -58,7 +58,9 @@ level_1_maps = sunpy.map.Map(files) # We get the pointing table outside of the loop for the relevant time range. # Otherwise you're making a call to the JSOC every single time. -pointing_table = get_pointing_table("jsoc", start=level_1_maps[0].date - 3 * u.h, end=level_1_maps[-1].date + 3 * u.h) +pointing_table = get_pointing_table( + "jsoc", time_range=(level_1_maps[0].date - 3 * u.h, level_1_maps[-1].date + 3 * u.h) +) # The same applies for the correction table. correction_table = get_correction_table(source="jsoc") diff --git a/examples/update_header_keywords.py b/examples/update_header_keywords.py deleted file mode 100644 index f013492..0000000 --- a/examples/update_header_keywords.py +++ /dev/null @@ -1,81 +0,0 @@ -""" -============================================= -Updating pointing keywords in the FITS header -============================================= - -This example demonstrates how to update the metadata in -an AIA FITS file to ensure that it has the most accurate -information regarding the spacecraft pointing. -""" - -import matplotlib.pyplot as plt - -import astropy.units as u - -import sunpy.map - -import aiapy.data.sample as sample_data -from aiapy.calibrate import update_pointing -from aiapy.calibrate.util import get_pointing_table - -############################################################################### -# An AIA FITS header contains various pieces of -# `standard `_. -# metadata that are critical to the physical interpretation of the data. -# These include the pointing of the spacecraft, necessary for connecting -# positions on the pixel grid to physical locations on the Sun, as well as -# the observer (i.e., satellite) location. -# -# While this metadata is recorded in the FITS header, some values in -# the headers exported by data providers (e.g. -# `Joint Science Operations Center (JSOC) `_ and -# the `Virtual Solar Observatory `_ -# may not always be the most accurate. In the case of the spacecraft -# pointing, a more accurate 3-hourly pointing table is available from the -# JSOC. -# -# For this example, we will read a 171 Å image from the aiapy sample data -# into a `~sunpy.map.Map` object. - -aia_map = sunpy.map.Map(sample_data.AIA_171_IMAGE) - -############################################################################### -# To update the pointing keywords, we can pass our `~sunpy.map.Map` to the -# `aiapy.calibrate.update_pointing` function. -# One needs to get the pointing information from the JSOC using the -# `aiapy.calibrate.util.get_pointing_table` function first. - -# Make range wide enough to get closest 3-hour pointing -pointing_table = get_pointing_table("JSOC", start=aia_map.date - 12 * u.h, end=aia_map.date + 12 * u.h) -aia_map_updated_pointing = update_pointing(aia_map, pointing_table=pointing_table) - -############################################################################### -# If we inspect the reference pixel and rotation matrix of the original map: - -print(aia_map.reference_pixel) -print(aia_map.rotation_matrix) - -############################################################################### -# and the map with the updated pointing information: - -print(aia_map_updated_pointing.reference_pixel) -print(aia_map_updated_pointing.rotation_matrix) - -############################################################################### -# We find that the relevant keywords, ``CRPIX1``, ``CRPIX2``, ``CDELT1``, ``CDELT2``, -# and ``CROTA2``, have been updated. -# -# Similarly, the Heliographic Stonyhurst (HGS) coordinates of the observer -# location in the header are correct. - -print(aia_map_updated_pointing.meta["hgln_obs"]) -print(aia_map_updated_pointing.meta["hglt_obs"]) - -############################################################################### -# Finally, plot the fixed map. - -fig = plt.figure() -ax = fig.add_subplot(projection=aia_map_updated_pointing) -aia_map_updated_pointing.plot(axes=ax) - -plt.show() From fd18fb9d1d5b10aff2a64ddc585fa7959663ce8e Mon Sep 17 00:00:00 2001 From: Nabil Freij Date: Wed, 8 Jan 2025 13:44:08 -0800 Subject: [PATCH 11/11] Apply suggestions from code review Co-authored-by: Will Barnes --- aiapy/calibrate/tests/test_prep.py | 1 + aiapy/calibrate/tests/test_util.py | 3 +- aiapy/calibrate/util.py | 50 ++++++++++++++++-------------- 3 files changed, 29 insertions(+), 25 deletions(-) diff --git a/aiapy/calibrate/tests/test_prep.py b/aiapy/calibrate/tests/test_prep.py index 14396a3..0240b1f 100644 --- a/aiapy/calibrate/tests/test_prep.py +++ b/aiapy/calibrate/tests/test_prep.py @@ -105,6 +105,7 @@ def test_register_level_15(lvl_15_map) -> None: pytest.param("jsoc", marks=pytest.mark.remote_data), pytest.param("SsW", marks=pytest.mark.remote_data), get_test_filepath("aia_V8_20171210_050627_response_table.txt"), + str(get_test_filepath("aia_V8_20171210_050627_response_table.txt")), ], ) def test_correct_degradation(aia_171_map, source) -> None: diff --git a/aiapy/calibrate/tests/test_util.py b/aiapy/calibrate/tests/test_util.py index 81cbc50..0e7b40c 100644 --- a/aiapy/calibrate/tests/test_util.py +++ b/aiapy/calibrate/tests/test_util.py @@ -121,6 +121,7 @@ def test_pointing_table_unavailable() -> None: [ pytest.param("SSW", marks=pytest.mark.remote_data), get_test_filepath("aia_V3_error_table.txt"), + str(get_test_filepath("aia_V3_error_table.txt")), ], ) def test_error_table(error_table) -> None: @@ -130,5 +131,5 @@ def test_error_table(error_table) -> None: def test_invalid_error_table_input() -> None: - with pytest.raises(TypeError, match="source must be a pathlib.Path, or 'SSW', not -1"): + with pytest.raises(TypeError, match="source must be a filepath, or 'SSW', not -1"): get_error_table(-1) diff --git a/aiapy/calibrate/util.py b/aiapy/calibrate/util.py index 0d98223..7734b37 100644 --- a/aiapy/calibrate/util.py +++ b/aiapy/calibrate/util.py @@ -47,11 +47,18 @@ "0a3f2db39d05c44185f6fdeec928089fb55d1ce1e0a805145050c6356cbc6e98", ) } +_URL_HASH_CORRECTION_TABLE["latest"] = _URL_HASH_CORRECTION_TABLE[10] +_URL_HASH_ERROR_TABLE["latest"] = _URL_HASH_ERROR_TABLE[3] -@manager.require("correction_table_v10", *_URL_HASH_CORRECTION_TABLE[10]) -def _fetch_correction_table_v10(): - return manager.get("correction_table_v10") +@manager.require("correction_table_latest", *_URL_HASH_CORRECTION_TABLE["latest"]) +def _fetch_correction_table_latest(): + return manager.get("correction_table_latest") + + +@manager.require("error_table_latest", *_URL_HASH_ERROR_TABLE["latest"]) +def _fetch_error_table_latest(): + return manager.get("error_table_latest") def get_correction_table(source): @@ -69,9 +76,9 @@ def get_correction_table(source): Parameters ---------- source: pathlib.Path, str - The source of the correction table. If it is a `pathlib.Path`, it must be a file. - A string file path will error as an invalid source. - If source is a string, it must either be "JSOC" which will fetch the most recent version from the JSOC or "SSW" which will fetch the most recent version (V10) from SSW. + The source of the correction table. + It can be a string or `pathlib.Path` for a file . + Otherwise, it must either be "JSOC" which will fetch the most recent version from the JSOC or "SSW" which will fetch the most recent version from SSW. Returns ------- @@ -82,10 +89,8 @@ def get_correction_table(source): -------- aiapy.calibrate.degradation """ - if isinstance(source, pathlib.Path): - table = QTable(astropy.io.ascii.read(source)) - elif isinstance(source, str) and source.lower() == "ssw": - table = QTable(astropy.io.ascii.read(_fetch_correction_table_v10())) + if isinstance(source, str) and source.lower() == "ssw": + table = QTable(astropy.io.ascii.read(_fetch_correction_table_latest())) elif isinstance(source, str) and source.lower() == "jsoc": # NOTE: the [!1=1!] disables the drms PrimeKey logic and enables # the query to find records that are ordinarily considered @@ -93,6 +98,8 @@ def get_correction_table(source): # and T_START. Without the !1=1! the query only returns the # latest record for each unique combination of those keywords. table = QTable.from_pandas(_get_data_from_jsoc(query="aia.response[][!1=1!]", key="**ALL**")) + elif isinstance(source, pathlib.Path | str): + table = QTable(astropy.io.ascii.read(source)) else: msg = f"correction_table must be a file path (pathlib.Path), 'JSOC' or 'SSW'. Not {source}" raise ValueError(msg) @@ -146,8 +153,9 @@ def _select_epoch_from_correction_table(channel: u.angstrom, obstime, correction table.sort("DATE") # Newest entries will be last if len(table) == 0: extra_msg = " Max version is 3." if channel == 4500 * u.AA else "" + msg = f"Correction table does not contain calibration for {channel}.{extra_msg}" raise ValueError( - f"Correction table does not contain calibration for {channel}." + extra_msg, + msg, ) # Select the epoch for the given observation time obstime_in_epoch = np.logical_and(obstime >= table["T_START"], obstime < table["T_STOP"]) @@ -250,11 +258,6 @@ def get_pointing_table(source, *, time_range=None): return table -@manager.require("error_table_v3", *_URL_HASH_ERROR_TABLE[3]) -def _fetch_error_table_v3(): - return manager.get("error_table_v3") - - def get_error_table(source="SSW") -> QTable: """ Fetches the error table from a SSW mirror or uses a local file if one is @@ -263,10 +266,9 @@ def get_error_table(source="SSW") -> QTable: Parameters ---------- source : pathlib.Path, str, optional - If input is a pathlib.Path, it is assumed to be a file path to a local error table. - If a path is provided as a string, it will error as an invalid source. - Otherwise, the input is allowed to be "SSW" (the default) which will - fetch the most recent version (V3) from SSW. + The input is allowed to be "SSW" (the default) which will + fetch the most recent version from SSW. + Otherwise, it must be a file path. Returns ------- @@ -278,12 +280,12 @@ def get_error_table(source="SSW") -> QTable: TypeError If ``error_table`` is not a file path. """ - if isinstance(source, pathlib.Path): + if isinstance(source, str) and source.lower() == "ssw": + error_table = QTable(astropy.io.ascii.read(_fetch_error_table_latest())) + elif isinstance(source, pathlib.Path | str): error_table = QTable(astropy.io.ascii.read(source)) - elif isinstance(source, str) and source.lower() == "ssw": - error_table = QTable(astropy.io.ascii.read(_fetch_error_table_v3())) else: - msg = f"source must be a pathlib.Path, or 'SSW', not {source}" + msg = f"source must be a filepath, or 'SSW', not {source}" raise TypeError(msg) error_table["DATE"] = Time(error_table["DATE"], scale="utc") error_table["T_START"] = Time(error_table["T_START"], scale="utc")