[WIP] BOEnv

dacbench/benchmarks/__init__.py

@@ -51,3 +51,15 @@
         "Theory Benchmark not installed. If you want to use this benchmark, "
         "please follow the installation guide."
     )
+
+dacbo_spec = importlib.util.find_spec("dacboenv")
+found = dacbo_spec is not None
+if found:
+    from dacbench.benchmarks.dacbo_benchmark import DACBOBenchmark
+
+    __all__.append("DACBOBenchmark")
+else:
+    warnings.warn(  # noqa: B028
+        "DACBOEnv not installed. If you want to use this benchmark, "
+        "please follow the installation guide."
+    )

dacbench/benchmarks/dacbo_benchmark.py

@@ -0,0 +1,149 @@
+"""DACBOEnv Benchmark."""
+
+from __future__ import annotations
+
+from importlib.resources import files
+from itertools import product
+from pathlib import Path
+
+import dacboenv
+import numpy as np
+import yaml
+from dacboenv.env.action import AcqParameterActionSpace
+from omegaconf import OmegaConf
+
+from dacbench.abstract_benchmark import AbstractBenchmark, objdict
+from dacbench.envs.dacbo import DACBOEnv
+
+
+def load_default_optimizer():
+    """Handles dacboenv configs to configure WEI as default."""
+    dacboenv_path = files("dacboenv")
+    base = OmegaConf.load(dacboenv_path / "configs/env/opt/base.yaml")
+    base.dacboenv.optimizer_cfg.smac_cfg.smac_kwargs = None
+    override = OmegaConf.load(
+        dacboenv_path / "configs/env/action/wei_alpha_continuous.yaml"
+    )
+    cfg = OmegaConf.merge(base, override)
+    cfg = OmegaConf.create({"optimizer": cfg.dacboenv.optimizer_cfg})
+
+    def replace_refs(node):
+        if isinstance(node, str):
+            return node.replace("dacboenv.optimizer_cfg", "optimizer")
+        if isinstance(node, dict):
+            return {k: replace_refs(v) for k, v in node.items()}
+        if isinstance(node, list):
+            return [replace_refs(v) for v in node]
+        return node
+
+    cfg = OmegaConf.create(replace_refs(OmegaConf.to_container(cfg, resolve=False)))
+    cfg.outdir = "runs/SMAC-DACBO/${benchmark_id}/${task_id}/${seed}"
+
+    return cfg
+
+
+INFO = {
+    "identifier": "DACBO",
+    "name": "DACBO",
+    "reward": f"""Default: [symlogregret]. Other options: {[
+        rew.name for rew in dacboenv.env.reward.ALL_REWARDS
+    ]}""",
+    "state_description": f"""Default: {[
+            "ubr_difference",
+            "acq_value_EI",
+            "acq_value_PI",
+            "previous_param",
+        ]}. Other options: {[
+        obs.name for obs in dacboenv.env.observation.ALL_OBSERVATIONS
+    ]}""",
+}
+
+DACBO_DEFAULTS = objdict(
+    {
+        "reward_range": [-np.inf, np.inf],
+        "seed": 0,
+        "instance_set_path": "bbob_2_default.yaml",
+        "optimizer_cfg": load_default_optimizer(),
+        "observation_keys": [
+            "ubr_difference",
+            "acq_value_EI",
+            "acq_value_PI",
+            "previous_param",
+        ],
+        "action_space_class": AcqParameterActionSpace,
+        "action_space_kwargs": {"bounds": [0, 1], "adjustment_type": "continuous"},
+        "reward_keys": ["symlogregret"],
+        "benchmark_info": INFO,
+    }
+)
+
+
+class DACBOBenchmark(AbstractBenchmark):
+    """DACBOEnv benchmark."""
+
+    def __init__(self, config_path=None, config=None):
+        """Init DACBOEnv benchmark."""
+        super().__init__(config_path, config)
+
+        if not self.config:
+            self.config = objdict(DACBO_DEFAULTS.copy())
+
+        for key in DACBO_DEFAULTS:
+            if key not in self.config:
+                self.config[key] = DACBO_DEFAULTS[key]
+
+    def get_environment(self):
+        """Returns the internal env."""
+        if "instance_set" not in self.config:
+            self.read_instance_set()
+        if "test_set" not in self.config and "test_set_path" in self.config:
+            self.read_instance_set(test=True)
+        env = DACBOEnv(self.config)
+        for func in self.wrap_funcs:
+            env = func(env)
+        return env
+
+    def read_instance_set(self, test=False):
+        """Reads the instance set."""
+        path_key = "test_set_path" if test else "instance_set_path"
+        set_key = "test_set" if test else "instance_set"
+        assert self.config[path_key]
+        instance_set_path = self.config[path_key]
+        try:  # Look in hydra search path if user uses hydra
+            from hydra.core.hydra_config import HydraConfig
+            config = HydraConfig.get()
+            hydra_candidate_paths = [
+                Path(path_description["path"]) / instance_set_path
+                for path_description in config["runtime"]["config_sources"]
+                if path_description["schema"] == "file"
+            ]
+            matched_hydra_files = list(
+                filter(lambda f: f.is_file(), hydra_candidate_paths)
+            )
+        except (ImportError, ValueError):
+            matched_hydra_files = []
+        if Path(instance_set_path).is_file():
+            path = instance_set_path
+        elif len(matched_hydra_files) > 0:
+            path = matched_hydra_files[0]
+        else:
+            path = (
+                Path(__file__).resolve().parent
+                / "../instance_sets/dacbo/"
+                / instance_set_path
+            )
+
+        with open(path) as f:
+            instance_data = yaml.safe_load(f)
+        self.config["task_ids"] = instance_data["task_ids"]
+        self.config["inner_seeds"] = instance_data.get("inner_seeds", None)
+        self.config[set_key] = dict(
+            enumerate(
+                product(
+                    instance_data.get("inner_seeds", [None]),
+                    instance_data["task_ids"],
+                )
+            )
+        )
+
+        assert len(self.config[set_key]) > 0, "ERROR: empty instance set"

dacbench/envs/__init__.py

@@ -48,3 +48,15 @@
         "SGD Benchmark not installed. If you want to use this benchmark, "
         "please follow the installation guide."
     )
+
+dacboenv_spec = importlib.util.find_spec("dacboenv")
+found = dacboenv_spec is not None
+if found:
+    from dacbench.envs.dacbo import DACBOEnv
+
+    __all__.append("DACBOEnv")
+else:
+    warnings.warn(  # noqa: B028
+        "DACBO Env not installed. If you want to use this benchmark, "
+        "please follow the installation guide."
+    )

dacbench/envs/dacbo.py

@@ -0,0 +1,40 @@
+"""DACBO Env."""
+
+from __future__ import annotations
+
+import numpy as np
+from dacboenv.dacboenv import DACBOEnv as DEnv
+from dacboenv.env.instance import ExternalInstanceSelector
+
+from dacbench.abstract_env import AbstractEnv
+
+
+class DACBOEnv(AbstractEnv):
+    """DACBO env."""
+
+    def __init__(self, config):
+        """Init DACBO env."""
+        config["instance_selector_class"] = ExternalInstanceSelector
+        self._env = DEnv(**config)
+        self._env.reset()  # Init spaces (NOT self.reset())
+        config["cutoff"] = np.inf
+        config["observation_space"] = self._env.observation_space
+        config["action_space"] = self._env.action_space
+        super().__init__(config)
+
+    def step(self, action):
+        """Takes one env step."""
+        super().step_()
+        state, reward, terminated, truncated, info = self._env.step(action)
+        return state, reward, terminated, truncated, info
+
+    def reset(self, seed=None, options=None):
+        """Resets the internal env."""
+        if options is None:
+            options = {}
+        super().reset_(seed, options)  # AbstractEnv picks next instance
+        self._env.instance_selector.set_instance(self.instance)
+        obs, info = self._env.reset()
+        self.observation_space = self._env.observation_space
+        self.action_space = self._env.action_space
+        return obs, info

dacbench/instance_sets/dacbo/bbob_2_default.yaml

@@ -0,0 +1,25 @@
+task_ids:
+  - "bbob/2/1/0"
+  - "bbob/2/2/0"
+  - "bbob/2/3/0"
+  - "bbob/2/4/0"
+  - "bbob/2/5/0"
+  - "bbob/2/6/0"
+  - "bbob/2/7/0"
+  - "bbob/2/8/0"
+  - "bbob/2/9/0"
+  - "bbob/2/10/0"
+  - "bbob/2/11/0"
+  - "bbob/2/12/0"
+  - "bbob/2/13/0"
+  - "bbob/2/14/0"
+  - "bbob/2/15/0"
+  - "bbob/2/16/0"
+  - "bbob/2/17/0"
+  - "bbob/2/18/0"
+  - "bbob/2/19/0"
+  - "bbob/2/20/0"
+inner_seeds:
+  - 1
+  - 2
+  - 3