From 959799b948dd7d11d72ea417d47f0c5f60e35fc2 Mon Sep 17 00:00:00 2001
From: Oleksandr <maksymets.o@gmail.com>
Date: Fri, 14 Jun 2019 22:00:44 -0700
Subject: [PATCH] Added description how to use config system, fixed missing
copyright headers, improved example and isort CI (#125)
* Added description how to use config system related to #20.
* Added missing copyright headers
* Improve registry example to be cleaner
* Adde habitat_sim as first party lib to isort CI setup
---
.circleci/config.yml | 4 +-
configs/README.md | 103 ++++++++++++++++++
docs/source/conf.py | 5 +
examples/register_new_sensors_and_measures.py | 15 ++-
.../datasets/pointnav/pointnav_generator.py | 6 +
habitat/datasets/utils.py | 6 +
habitat_baselines/README.md | 3 +-
habitat_baselines/agents/slam_agents.py | 6 +
test/test_demo_notebook.py | 6 +-
test/test_relative_camera.py | 6 +-
10 files changed, 150 insertions(+), 10 deletions(-)
create mode 100644 configs/README.md
diff --git a/.circleci/config.yml b/.circleci/config.yml
index 76112c330..206edb016 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -25,8 +25,8 @@ jobs:
- run:
name: run isort
command: |
- isort . -rc --multi-line 3 --trailing-comma --force-grid-wrap 0 --line-width 79 --combine-as -o torch -o pytest -o torchvision -o matplotlib --diff
- isort . -rc --multi-line 3 --trailing-comma --force-grid-wrap 0 --line-width 79 --combine-as -o torch -o pytest -o torchvision -o matplotlib --check-only
+ isort . -rc --multi-line 3 --trailing-comma --force-grid-wrap 0 --line-width 79 --combine-as -o torch -o pytest -o torchvision -o matplotlib -p habitat_sim --diff
+ isort . -rc --multi-line 3 --trailing-comma --force-grid-wrap 0 --line-width 79 --combine-as -o torch -o pytest -o torchvision -o matplotlib -p habitat_sim --check-only
install_and_test_ubuntu:
<<: *gpu
diff --git a/configs/README.md b/configs/README.md
new file mode 100644
index 000000000..c564d4d9f
--- /dev/null
+++ b/configs/README.md
@@ -0,0 +1,103 @@
+Habitat-API Configuration
+==============================
+
+Habitat-API uses [Yacs configuration system](https://github.com/rbgirshick/yacs)
+with the paradigm of `your code + a YACS config for experiment E (+
+external dependencies + hardware + other nuisance terms ...) =
+reproducible experiment E`. Yacs advantages:
+- Checks for type consistency.
+- All parameters and default values are searchable in the code.
+- A parameter doesn't need to be set always as each parameter has a
+ default value.
+- Ability to freeze config to prevent unintended changes.
+
+## Config usage
+An example of how to merge default config with 2 others configs and overwrite
+one parameter that could come from the command line:
+```
+ merged_config = get_config(
+ config_paths=["configs/tasks/pointnav.yaml",
+ "configs/dataset/val.yaml"],
+ opts=["ENVIRONMENT.MAX_EPISODE_STEPS", steps_limit]
+ )
+
+```
+
+## Config structure
+Below is the structure of config used for Habitat:
+- Environment
+- Task
+ - Sensors
+ - Measurements
+- Simulator
+ - Agent
+ - Sensors
+- Dataset
+
+We use node names (e.g. `SENSORS: ['RGB_SENSOR', 'DEPTH_SENSOR']`) instead of list
+of config nodes (e.g. `SENSORS: [{TYPE = "HabitatSimDepthSensor",
+MIN_DEPTH = 0}, ...]`) to declare the Sensors attached to an Agent or Measures
+enabled for the Task . With this approach, it's still easy to overwrite a
+particular sensor parameter in yaml file without redefining the whole sensor
+config.
+
+## Extending the config
+Example of how to extend a config outside of `habtiat-api` repository.
+First, we create a config extending the default config in the code and re-use
+`habitat.get_config()`:
+```
+import habitat
+import argparse
+from typing import List, Optional, Union
+
+_C = habitat.get_config()
+_C.defrost()
+# Add new parameters to the config
+_C.TASK.EPISODE_INFO = habitat.Config()
+_C.TASK.EPISODE_INFO.TYPE = "EpisodeInfo"
+_C.TASK.EPISODE_INFO.VALUE = 5
+_C.TASK.MEASUREMENTS.append("EPISODE_INFO")
+
+# New function returning extended Habitat config that should be used instead
+# of habitat.get_config()
+def my_get_config(
+ config_paths: Optional[Union[List[str], str]] = None,
+ opts: Optional[list] = None,
+) -> habitat.Config:
+ CONFIG_FILE_SEPARATOR = ","
+ config = _C.clone()
+ if config_paths:
+ if isinstance(config_paths, str):
+ if CONFIG_FILE_SEPARATOR in config_paths:
+ config_paths = config_paths.split(CONFIG_FILE_SEPARATOR)
+ else:
+ config_paths = [config_paths]
+
+ for config_path in config_paths:
+ config.merge_from_file(config_path)
+
+ if opts:
+ config.merge_from_list(opts)
+
+ config.freeze()
+ return config
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument(
+ "--task-config",
+ type=str,
+ default="configs/tasks/pointnav.yaml,"
+ "configs/datasets/pointnav/habitat_test.yaml",
+ )
+ parser.add_argument(
+ "opts",
+ default=None,
+ nargs=argparse.REMAINDER,
+ help="Modify config options from command line",
+ )
+ args = parser.parse_args()
+ config = my_get_config(config_paths=args.task_config, opts=args.opts)
+ env = habitat.Env(config)
+
+```
\ No newline at end of file
diff --git a/docs/source/conf.py b/docs/source/conf.py
index 44327f530..317c59a6f 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -1,4 +1,9 @@
#!/usr/bin/env python3
+
+# Copyright (c) Facebook, Inc. and its affiliates.
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
# -*- coding: utf-8 -*-
#
# habitat-api documentation build configuration file, created by
diff --git a/examples/register_new_sensors_and_measures.py b/examples/register_new_sensors_and_measures.py
index a7bb634cc..a857f1866 100644
--- a/examples/register_new_sensors_and_measures.py
+++ b/examples/register_new_sensors_and_measures.py
@@ -1,17 +1,22 @@
+#!/usr/bin/env python3
+
+# Copyright (c) Facebook, Inc. and its affiliates.
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
from typing import Any
import numpy as np
from gym import spaces
import habitat
-from habitat.config import Config as CN
# Define the measure and register it with habitat
# By default, the things are registered with the class name
@habitat.registry.register_measure
class EpisodeInfo(habitat.Measure):
- def __init__(self, sim, config):
+ def __init__(self, sim, config, **kwargs: Any):
# This measure only needs the config
self._config = config
@@ -38,7 +43,7 @@ class EpisodeInfo(habitat.Measure):
# For the sensor, we will register it with a custom name
@habitat.registry.register_sensor(name="my_supercool_sensor")
class AgentPositionSensor(habitat.Sensor):
- def __init__(self, sim, config):
+ def __init__(self, sim, config, **kwargs: Any):
super().__init__(config=config)
self._sim = sim
@@ -73,7 +78,7 @@ def main():
config.defrost()
# Add things to the config to for the measure
- config.TASK.EPISODE_INFO = CN()
+ config.TASK.EPISODE_INFO = habitat.Config()
# The type field is used to look-up the measure in the registry.
# By default, the things are registered with the class name
config.TASK.EPISODE_INFO.TYPE = "EpisodeInfo"
@@ -82,7 +87,7 @@ def main():
config.TASK.MEASUREMENTS.append("EPISODE_INFO")
# Now define the config for the sensor
- config.TASK.AGENT_POSITION_SENSOR = CN()
+ config.TASK.AGENT_POSITION_SENSOR = habitat.Config()
# Use the custom name
config.TASK.AGENT_POSITION_SENSOR.TYPE = "my_supercool_sensor"
config.TASK.AGENT_POSITION_SENSOR.ANSWER_TO_LIFE = 42
diff --git a/habitat/datasets/pointnav/pointnav_generator.py b/habitat/datasets/pointnav/pointnav_generator.py
index 8b08174be..047876adf 100644
--- a/habitat/datasets/pointnav/pointnav_generator.py
+++ b/habitat/datasets/pointnav/pointnav_generator.py
@@ -1,3 +1,9 @@
+#!/usr/bin/env python3
+
+# Copyright (c) Facebook, Inc. and its affiliates.
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
from typing import Optional
import numpy as np
diff --git a/habitat/datasets/utils.py b/habitat/datasets/utils.py
index 918096a53..67395525c 100644
--- a/habitat/datasets/utils.py
+++ b/habitat/datasets/utils.py
@@ -1,3 +1,9 @@
+#!/usr/bin/env python3
+
+# Copyright (c) Facebook, Inc. and its affiliates.
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
from typing import List
from habitat.core.logging import logger
diff --git a/habitat_baselines/README.md b/habitat_baselines/README.md
index 242d07e09..f3150a210 100644
--- a/habitat_baselines/README.md
+++ b/habitat_baselines/README.md
@@ -76,4 +76,5 @@ Set argument `--task-config` to `tasks/pointnav_mp3d.yaml` for training on [Matt
**SLAM based**
-- [Handcrafted agent baseline adopted from the paper "Benchmarking Classic and Learned Navigation in Complex 3D Environments"](habitat_baselines/slambased/README.md)
+- [Handcrafted agent baseline](slambased/README.md) adopted from the paper
+"Benchmarking Classic and Learned Navigation in Complex 3D Environments".
diff --git a/habitat_baselines/agents/slam_agents.py b/habitat_baselines/agents/slam_agents.py
index b7694e826..846584062 100644
--- a/habitat_baselines/agents/slam_agents.py
+++ b/habitat_baselines/agents/slam_agents.py
@@ -1,3 +1,9 @@
+#!/usr/bin/env python3
+
+# Copyright (c) Facebook, Inc. and its affiliates.
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
import argparse
import os
import random
diff --git a/test/test_demo_notebook.py b/test/test_demo_notebook.py
index fe756c436..ca4e5ad23 100644
--- a/test/test_demo_notebook.py
+++ b/test/test_demo_notebook.py
@@ -1,4 +1,8 @@
-import os
+#!/usr/bin/env python3
+
+# Copyright (c) Facebook, Inc. and its affiliates.
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
import pytest
diff --git a/test/test_relative_camera.py b/test/test_relative_camera.py
index c85b1cbdb..fd87dff85 100644
--- a/test/test_relative_camera.py
+++ b/test/test_relative_camera.py
@@ -1,4 +1,8 @@
-import os
+#!/usr/bin/env python3
+
+# Copyright (c) Facebook, Inc. and its affiliates.
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
import pytest
--
GitLab