Skip to content
Snippets Groups Projects
Commit dd75f18f authored by JasonJiazhiZhang's avatar JasonJiazhiZhang Committed by Oleksandr
Browse files

Move circle-ci experiment testing to pytest (#176)

With the recent refactor/redesign of habitat_baselines, all experiment parameters can be specified through config files. Thus it makes sense to turn the python script run in circle-ci into a separate test file in our test/. This also makes it more extensible when we add more trainers in the future.
parent 398dcdd3
No related branches found
No related tags found
No related merge requests found
......@@ -196,7 +196,6 @@ jobs:
. activate habitat; cd habitat-api
python setup.py develop --all
python setup.py test
python -u habitat_baselines/run.py --exp-config habitat_baselines/config/pointnav/ppo_train_test.yaml --run-type train
workflows:
......
......@@ -5,11 +5,11 @@ SIMULATOR_GPU_ID: 0
TORCH_GPU_ID: 0
VIDEO_OPTION: []
TENSORBOARD_DIR: ""
EVAL_CKPT_PATH_DIR: "data/new_checkpoints"
EVAL_CKPT_PATH_DIR: "data/test_checkpoints/ppo/pointnav/ckpt.0.pth"
NUM_PROCESSES: 1
CHECKPOINT_FOLDER: "data/new_checkpoints"
CHECKPOINT_FOLDER: "data/test_checkpoints/ppo/pointnav/"
NUM_UPDATES: 10
LOG_INTERVAL: 1
LOG_INTERVAL: 100
CHECKPOINT_INTERVAL: 1
RL:
......
......@@ -33,8 +33,24 @@ def main():
nargs=argparse.REMAINDER,
help="Modify config options from command line",
)
args = parser.parse_args()
config = get_config(args.exp_config, args.opts)
run_exp(**vars(args))
def run_exp(exp_config: str, run_type: str, opts=None) -> None:
r"""Runs experiment given mode and config
Args:
exp_config: path to config file.
run_type: "train" or "eval.
opts: list of strings of additional config options.
Returns:
None.
"""
config = get_config(exp_config, opts)
random.seed(config.TASK_CONFIG.SEED)
np.random.seed(config.TASK_CONFIG.SEED)
......@@ -42,9 +58,9 @@ def main():
assert trainer_init is not None, f"{config.TRAINER_NAME} is not supported"
trainer = trainer_init(config)
if args.run_type == "train":
if run_type == "train":
trainer.train()
elif args.run_type == "eval":
elif run_type == "eval":
trainer.eval()
......
#!/usr/bin/env python3
# Copyright (c) Facebook, Inc. and its affiliates.
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
from glob import glob
import pytest
try:
from habitat_baselines.run import run_exp
from habitat_baselines.common.base_trainer import BaseRLTrainer
from habitat_baselines.config.default import get_config
baseline_installed = True
except ImportError:
baseline_installed = False
@pytest.mark.skipif(
not baseline_installed, reason="baseline sub-module not installed"
)
@pytest.mark.parametrize(
"test_cfg_path,mode",
[
(cfg, mode)
for mode in ("train", "eval")
for cfg in glob("habitat_baselines/config/test/*")
],
)
def test_trainers(test_cfg_path, mode):
run_exp(test_cfg_path, mode)
@pytest.mark.skipif(
not baseline_installed, reason="baseline sub-module not installed"
)
def test_eval_config():
ckpt_opts = ["VIDEO_OPTION", "[]"]
eval_opts = ["VIDEO_OPTION", "['disk']"]
ckpt_cfg = get_config(None, ckpt_opts)
assert ckpt_cfg.VIDEO_OPTION == []
assert ckpt_cfg.CMD_TRAILING_OPTS == ["VIDEO_OPTION", "[]"]
eval_cfg = get_config(None, eval_opts)
assert eval_cfg.VIDEO_OPTION == ["disk"]
assert eval_cfg.CMD_TRAILING_OPTS == ["VIDEO_OPTION", "['disk']"]
trainer = BaseRLTrainer(get_config())
assert trainer.config.VIDEO_OPTION == ["disk", "tensorboard"]
returned_config = trainer._setup_eval_config(checkpoint_config=ckpt_cfg)
assert returned_config.VIDEO_OPTION == []
trainer = BaseRLTrainer(eval_cfg)
returned_config = trainer._setup_eval_config(ckpt_cfg)
assert returned_config.VIDEO_OPTION == ["disk"]
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment