Move circle-ci experiment testing to pytest (#176)

With the recent refactor/redesign of habitat_baselines, all experiment parameters can be specified through config files. Thus it makes sense to turn the python script run in circle-ci into a separate test file in our test/. This also makes it more extensible when we add more trainers in the future.

.circleci/config.yml

@@ -196,7 +196,6 @@ jobs:
               . activate habitat; cd habitat-api
               python setup.py develop --all
               python setup.py test
-              python -u habitat_baselines/run.py --exp-config habitat_baselines/config/pointnav/ppo_train_test.yaml --run-type train
 
 
 workflows:

habitat_baselines/config/pointnav/ppo_train_test.yaml → habitat_baselines/config/test/ppo_pointnav_test.yaml

@@ -5,11 +5,11 @@ SIMULATOR_GPU_ID: 0
 TORCH_GPU_ID: 0
 VIDEO_OPTION: []
 TENSORBOARD_DIR: ""
-EVAL_CKPT_PATH_DIR: "data/new_checkpoints"
+EVAL_CKPT_PATH_DIR: "data/test_checkpoints/ppo/pointnav/ckpt.0.pth"
 NUM_PROCESSES: 1
-CHECKPOINT_FOLDER: "data/new_checkpoints"
+CHECKPOINT_FOLDER: "data/test_checkpoints/ppo/pointnav/"
 NUM_UPDATES: 10
-LOG_INTERVAL: 1
+LOG_INTERVAL: 100
 CHECKPOINT_INTERVAL: 1
 
 RL:

habitat_baselines/run.py

@@ -33,8 +33,24 @@ def main():
         nargs=argparse.REMAINDER,
         help="Modify config options from command line",
     )
+
     args = parser.parse_args()
-    config = get_config(args.exp_config, args.opts)
+    run_exp(**vars(args))
+
+
+def run_exp(exp_config: str, run_type: str, opts=None) -> None:
+    r"""Runs experiment given mode and config
+
+    Args:
+        exp_config: path to config file.
+        run_type: "train" or "eval.
+        opts: list of strings of additional config options.
+
+    Returns:
+        None.
+    """
+    config = get_config(exp_config, opts)
+
     random.seed(config.TASK_CONFIG.SEED)
     np.random.seed(config.TASK_CONFIG.SEED)
 
@@ -42,9 +58,9 @@ def main():
     assert trainer_init is not None, f"{config.TRAINER_NAME} is not supported"
     trainer = trainer_init(config)
 
-    if args.run_type == "train":
+    if run_type == "train":
         trainer.train()
-    elif args.run_type == "eval":
+    elif run_type == "eval":
         trainer.eval()
 
 

test/test_baseline_trainers.py

+#!/usr/bin/env python3
+
+# Copyright (c) Facebook, Inc. and its affiliates.
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+from glob import glob
+
+import pytest
+
+try:
+    from habitat_baselines.run import run_exp
+    from habitat_baselines.common.base_trainer import BaseRLTrainer
+    from habitat_baselines.config.default import get_config
+
+    baseline_installed = True
+except ImportError:
+    baseline_installed = False
+
+
+@pytest.mark.skipif(
+    not baseline_installed, reason="baseline sub-module not installed"
+)
+@pytest.mark.parametrize(
+    "test_cfg_path,mode",
+    [
+        (cfg, mode)
+        for mode in ("train", "eval")
+        for cfg in glob("habitat_baselines/config/test/*")
+    ],
+)
+def test_trainers(test_cfg_path, mode):
+    run_exp(test_cfg_path, mode)
+
+
+@pytest.mark.skipif(
+    not baseline_installed, reason="baseline sub-module not installed"
+)
+def test_eval_config():
+    ckpt_opts = ["VIDEO_OPTION", "[]"]
+    eval_opts = ["VIDEO_OPTION", "['disk']"]
+
+    ckpt_cfg = get_config(None, ckpt_opts)
+    assert ckpt_cfg.VIDEO_OPTION == []
+    assert ckpt_cfg.CMD_TRAILING_OPTS == ["VIDEO_OPTION", "[]"]
+
+    eval_cfg = get_config(None, eval_opts)
+    assert eval_cfg.VIDEO_OPTION == ["disk"]
+    assert eval_cfg.CMD_TRAILING_OPTS == ["VIDEO_OPTION", "['disk']"]
+
+    trainer = BaseRLTrainer(get_config())
+    assert trainer.config.VIDEO_OPTION == ["disk", "tensorboard"]
+    returned_config = trainer._setup_eval_config(checkpoint_config=ckpt_cfg)
+    assert returned_config.VIDEO_OPTION == []
+
+    trainer = BaseRLTrainer(eval_cfg)
+    returned_config = trainer._setup_eval_config(ckpt_cfg)
+    assert returned_config.VIDEO_OPTION == ["disk"]