diff --git a/habitat/sims/habitat_simulator.py b/habitat/sims/habitat_simulator.py
index 30c4ca5a4e60bf74738a5cc6947cc1b74747c8e0..0a70cee0a0557c3caad5f55c378550af25be82fc 100644
--- a/habitat/sims/habitat_simulator.py
+++ b/habitat/sims/habitat_simulator.py
@@ -420,6 +420,13 @@ class HabitatSim(habitat.Simulator):
         state = self.get_agent_state(agent_id)
         state.position = position
         state.rotation = rotation
+
+        # NB: The agent state also contains the sensor states in _absolute_ coordinates.
+        # In order to set the agent's body to a specific location and have the sensors follow,
+        # we must not provide any state for the sensors.
+        # This will cause them to follow the agent's body
+        state.sensor_states = dict()
+
         agent.set_state(state, reset_sensors)
 
         self._check_agent_position(position, agent_id)