diff --git a/.coveragerc b/.coveragerc
index a4819a124c942a36315895cc107657d944b746f1..5fa34f2765683793d385b315dc982500e68b0c79 100644
--- a/.coveragerc
+++ b/.coveragerc
@@ -1009,7 +1009,6 @@ omit =
homeassistant/components/samsungtv/diagnostics.py
homeassistant/components/satel_integra/*
homeassistant/components/schluter/*
- homeassistant/components/scrape/sensor.py
homeassistant/components/screenlogic/__init__.py
homeassistant/components/screenlogic/binary_sensor.py
homeassistant/components/screenlogic/climate.py
diff --git a/CODEOWNERS b/CODEOWNERS
index b7b688a0e73c06e17d897f28974a779254360663..ceefad2bad42a56afe9d943577aa9f4bb9a4281e 100644
--- a/CODEOWNERS
+++ b/CODEOWNERS
@@ -808,6 +808,7 @@ homeassistant/components/scene/* @home-assistant/core
tests/components/scene/* @home-assistant/core
homeassistant/components/schluter/* @prairieapps
homeassistant/components/scrape/* @fabaff
+tests/components/scrape/* @fabaff
homeassistant/components/screenlogic/* @dieselrabbit @bdraco
tests/components/screenlogic/* @dieselrabbit @bdraco
homeassistant/components/script/* @home-assistant/core
diff --git a/homeassistant/components/scrape/sensor.py b/homeassistant/components/scrape/sensor.py
index c0523e4cbe2537e846543f175dd794691bab9e47..8f2a672ef06ab59f945051bddf9e30f3d192eb47 100644
--- a/homeassistant/components/scrape/sensor.py
+++ b/homeassistant/components/scrape/sensor.py
@@ -2,6 +2,7 @@
from __future__ import annotations
import logging
+from typing import Any
from bs4 import BeautifulSoup
import httpx
@@ -11,7 +12,7 @@ from homeassistant.components.rest.data import RestData
from homeassistant.components.sensor import (
CONF_STATE_CLASS,
DEVICE_CLASSES_SCHEMA,
- PLATFORM_SCHEMA,
+ PLATFORM_SCHEMA as PARENT_PLATFORM_SCHEMA,
STATE_CLASSES_SCHEMA,
SensorEntity,
)
@@ -33,6 +34,7 @@ from homeassistant.core import HomeAssistant
from homeassistant.exceptions import PlatformNotReady
import homeassistant.helpers.config_validation as cv
from homeassistant.helpers.entity_platform import AddEntitiesCallback
+from homeassistant.helpers.template import Template
from homeassistant.helpers.typing import ConfigType, DiscoveryInfoType
_LOGGER = logging.getLogger(__name__)
@@ -44,7 +46,7 @@ CONF_INDEX = "index"
DEFAULT_NAME = "Web scrape"
DEFAULT_VERIFY_SSL = True
-PLATFORM_SCHEMA = PLATFORM_SCHEMA.extend(
+PLATFORM_SCHEMA = PARENT_PLATFORM_SCHEMA.extend(
{
vol.Required(CONF_RESOURCE): cv.string,
vol.Required(CONF_SELECT): cv.string,
@@ -73,32 +75,32 @@ async def async_setup_platform(
discovery_info: DiscoveryInfoType | None = None,
) -> None:
"""Set up the Web scrape sensor."""
- name = config.get(CONF_NAME)
- resource = config.get(CONF_RESOURCE)
- method = "GET"
- payload = None
- headers = config.get(CONF_HEADERS)
- verify_ssl = config.get(CONF_VERIFY_SSL)
- select = config.get(CONF_SELECT)
- attr = config.get(CONF_ATTR)
- index = config.get(CONF_INDEX)
- unit = config.get(CONF_UNIT_OF_MEASUREMENT)
- device_class = config.get(CONF_DEVICE_CLASS)
- state_class = config.get(CONF_STATE_CLASS)
- username = config.get(CONF_USERNAME)
- password = config.get(CONF_PASSWORD)
-
- if (value_template := config.get(CONF_VALUE_TEMPLATE)) is not None:
+ name: str = config[CONF_NAME]
+ resource: str = config[CONF_RESOURCE]
+ method: str = "GET"
+ payload: str | None = None
+ headers: str | None = config.get(CONF_HEADERS)
+ verify_ssl: bool = config[CONF_VERIFY_SSL]
+ select: str | None = config.get(CONF_SELECT)
+ attr: str | None = config.get(CONF_ATTR)
+ index: int = config[CONF_INDEX]
+ unit: str | None = config.get(CONF_UNIT_OF_MEASUREMENT)
+ device_class: str | None = config.get(CONF_DEVICE_CLASS)
+ state_class: str | None = config.get(CONF_STATE_CLASS)
+ username: str | None = config.get(CONF_USERNAME)
+ password: str | None = config.get(CONF_PASSWORD)
+ value_template: Template | None = config.get(CONF_VALUE_TEMPLATE)
+
+ if value_template is not None:
value_template.hass = hass
- auth: httpx.DigestAuth | tuple[str, str] | None
+ auth: httpx.DigestAuth | tuple[str, str] | None = None
if username and password:
if config.get(CONF_AUTHENTICATION) == HTTP_DIGEST_AUTHENTICATION:
auth = httpx.DigestAuth(username, password)
else:
auth = (username, password)
- else:
- auth = None
+
rest = RestData(hass, method, resource, auth, headers, None, payload, verify_ssl)
await rest.async_update()
@@ -128,19 +130,19 @@ class ScrapeSensor(SensorEntity):
def __init__(
self,
- rest,
- name,
- select,
- attr,
- index,
- value_template,
- unit,
- device_class,
- state_class,
- ):
+ rest: RestData,
+ name: str,
+ select: str | None,
+ attr: str | None,
+ index: int,
+ value_template: Template | None,
+ unit: str | None,
+ device_class: str | None,
+ state_class: str | None,
+ ) -> None:
"""Initialize a web scrape sensor."""
self.rest = rest
- self._state = None
+ self._attr_native_value = None
self._select = select
self._attr = attr
self._index = index
@@ -150,12 +152,7 @@ class ScrapeSensor(SensorEntity):
self._attr_device_class = device_class
self._attr_state_class = state_class
- @property
- def native_value(self):
- """Return the state of the device."""
- return self._state
-
- def _extract_value(self):
+ def _extract_value(self) -> Any:
"""Parse the html extraction in the executor."""
raw_data = BeautifulSoup(self.rest.data, "html.parser")
_LOGGER.debug(raw_data)
@@ -180,30 +177,26 @@ class ScrapeSensor(SensorEntity):
_LOGGER.debug(value)
return value
- async def async_update(self):
+ async def async_update(self) -> None:
"""Get the latest data from the source and updates the state."""
await self.rest.async_update()
await self._async_update_from_rest_data()
- async def async_added_to_hass(self):
+ async def async_added_to_hass(self) -> None:
"""Ensure the data from the initial update is reflected in the state."""
await self._async_update_from_rest_data()
- async def _async_update_from_rest_data(self):
+ async def _async_update_from_rest_data(self) -> None:
"""Update state from the rest data."""
if self.rest.data is None:
_LOGGER.error("Unable to retrieve data for %s", self.name)
return
- try:
- value = await self.hass.async_add_executor_job(self._extract_value)
- except IndexError:
- _LOGGER.error("Unable to extract data from HTML for %s", self.name)
- return
+ value = await self.hass.async_add_executor_job(self._extract_value)
if self._value_template is not None:
- self._state = self._value_template.async_render_with_possible_json_value(
- value, None
+ self._attr_native_value = (
+ self._value_template.async_render_with_possible_json_value(value, None)
)
else:
- self._state = value
+ self._attr_native_value = value
diff --git a/requirements_test_all.txt b/requirements_test_all.txt
index 325b6f8f554cded54f356688198fa0b2c35cf63f..10bb8f5af4d4c29b80c785e4c537bc466520906d 100644
--- a/requirements_test_all.txt
+++ b/requirements_test_all.txt
@@ -275,6 +275,9 @@ azure-eventhub==5.7.0
# homeassistant.components.homekit
base36==0.1.1
+# homeassistant.components.scrape
+beautifulsoup4==4.10.0
+
# homeassistant.components.zha
bellows==0.29.0
diff --git a/tests/components/scrape/__init__.py b/tests/components/scrape/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..0ba9266a79d96423659b14beea59f935f191dcd8
--- /dev/null
+++ b/tests/components/scrape/__init__.py
@@ -0,0 +1,83 @@
+"""Tests for scrape component."""
+from __future__ import annotations
+
+from typing import Any
+
+
+def return_config(
+ select,
+ name,
+ *,
+ attribute=None,
+ index=None,
+ template=None,
+ uom=None,
+ device_class=None,
+ state_class=None,
+ authentication=None,
+ username=None,
+ password=None,
+ headers=None,
+) -> dict[str, dict[str, Any]]:
+ """Return config."""
+ config = {
+ "platform": "scrape",
+ "resource": "https://www.home-assistant.io",
+ "select": select,
+ "name": name,
+ }
+ if attribute:
+ config["attribute"] = attribute
+ if index:
+ config["index"] = index
+ if template:
+ config["value_template"] = template
+ if uom:
+ config["unit_of_measurement"] = uom
+ if device_class:
+ config["device_class"] = device_class
+ if state_class:
+ config["state_class"] = state_class
+ if authentication:
+ config["authentication"] = authentication
+ config["username"] = username
+ config["password"] = password
+ if headers:
+ config["headers"] = headers
+ return config
+
+
+class MockRestData:
+ """Mock RestData."""
+
+ def __init__(
+ self,
+ payload,
+ ):
+ """Init RestDataMock."""
+ self.data: str | None = None
+ self.payload = payload
+ self.count = 0
+
+ async def async_update(self, data: bool | None = True) -> None:
+ """Update."""
+ self.count += 1
+ if self.payload == "test_scrape_sensor":
+ self.data = (
+ "<div class='current-version material-card text'>"
+ "<h1>Current Version: 2021.12.10</h1>Released: <span class='release-date'>January 17, 2022</span>"
+ "<div class='links' style='links'><a href='/latest-release-notes/'>Release notes</a></div></div>"
+ "<template>Trying to get</template>"
+ )
+ if self.payload == "test_scrape_uom_and_classes":
+ self.data = (
+ "<div class='current-temp temp-card text'>"
+ "<h3>Current Temperature: 22.1</h3>"
+ "<div class='links'><a href='/check_temp/'>Temp check</a></div></div>"
+ )
+ if self.payload == "test_scrape_sensor_authentication":
+ self.data = "<div class='return'>secret text</div>"
+ if self.payload == "test_scrape_sensor_no_data":
+ self.data = None
+ if self.count == 3:
+ self.data = None
diff --git a/tests/components/scrape/test_sensor.py b/tests/components/scrape/test_sensor.py
new file mode 100644
index 0000000000000000000000000000000000000000..aaf156208efd34283e119b83637e4f1f52d8394b
--- /dev/null
+++ b/tests/components/scrape/test_sensor.py
@@ -0,0 +1,206 @@
+"""The tests for the Scrape sensor platform."""
+from __future__ import annotations
+
+from unittest.mock import patch
+
+from homeassistant.components.sensor import SensorDeviceClass, SensorStateClass
+from homeassistant.components.sensor.const import CONF_STATE_CLASS
+from homeassistant.const import (
+ CONF_DEVICE_CLASS,
+ CONF_UNIT_OF_MEASUREMENT,
+ STATE_UNKNOWN,
+ TEMP_CELSIUS,
+)
+from homeassistant.core import HomeAssistant
+from homeassistant.helpers.entity_component import async_update_entity
+from homeassistant.setup import async_setup_component
+
+from . import MockRestData, return_config
+
+DOMAIN = "scrape"
+
+
+async def test_scrape_sensor(hass: HomeAssistant) -> None:
+ """Test Scrape sensor minimal."""
+ config = {"sensor": return_config(select=".current-version h1", name="HA version")}
+
+ mocker = MockRestData("test_scrape_sensor")
+ with patch(
+ "homeassistant.components.scrape.sensor.RestData",
+ return_value=mocker,
+ ):
+ assert await async_setup_component(hass, "sensor", config)
+ await hass.async_block_till_done()
+
+ state = hass.states.get("sensor.ha_version")
+ assert state.state == "Current Version: 2021.12.10"
+
+
+async def test_scrape_sensor_value_template(hass: HomeAssistant) -> None:
+ """Test Scrape sensor with value template."""
+ config = {
+ "sensor": return_config(
+ select=".current-version h1",
+ name="HA version",
+ template="{{ value.split(':')[1] }}",
+ )
+ }
+
+ mocker = MockRestData("test_scrape_sensor")
+ with patch(
+ "homeassistant.components.scrape.sensor.RestData",
+ return_value=mocker,
+ ):
+ assert await async_setup_component(hass, "sensor", config)
+ await hass.async_block_till_done()
+
+ state = hass.states.get("sensor.ha_version")
+ assert state.state == "2021.12.10"
+
+
+async def test_scrape_uom_and_classes(hass: HomeAssistant) -> None:
+ """Test Scrape sensor for unit of measurement, device class and state class."""
+ config = {
+ "sensor": return_config(
+ select=".current-temp h3",
+ name="Current Temp",
+ template="{{ value.split(':')[1] }}",
+ uom="°C",
+ device_class="temperature",
+ state_class="measurement",
+ )
+ }
+
+ mocker = MockRestData("test_scrape_uom_and_classes")
+ with patch(
+ "homeassistant.components.scrape.sensor.RestData",
+ return_value=mocker,
+ ):
+ assert await async_setup_component(hass, "sensor", config)
+ await hass.async_block_till_done()
+
+ state = hass.states.get("sensor.current_temp")
+ assert state.state == "22.1"
+ assert state.attributes[CONF_UNIT_OF_MEASUREMENT] == TEMP_CELSIUS
+ assert state.attributes[CONF_DEVICE_CLASS] == SensorDeviceClass.TEMPERATURE
+ assert state.attributes[CONF_STATE_CLASS] == SensorStateClass.MEASUREMENT
+
+
+async def test_scrape_sensor_authentication(hass: HomeAssistant) -> None:
+ """Test Scrape sensor with authentication."""
+ config = {
+ "sensor": [
+ return_config(
+ select=".return",
+ name="Auth page",
+ username="user@secret.com",
+ password="12345678",
+ authentication="digest",
+ ),
+ return_config(
+ select=".return",
+ name="Auth page2",
+ username="user@secret.com",
+ password="12345678",
+ ),
+ ]
+ }
+
+ mocker = MockRestData("test_scrape_sensor_authentication")
+ with patch(
+ "homeassistant.components.scrape.sensor.RestData",
+ return_value=mocker,
+ ):
+ assert await async_setup_component(hass, "sensor", config)
+ await hass.async_block_till_done()
+
+ state = hass.states.get("sensor.auth_page")
+ assert state.state == "secret text"
+ state2 = hass.states.get("sensor.auth_page2")
+ assert state2.state == "secret text"
+
+
+async def test_scrape_sensor_no_data(hass: HomeAssistant) -> None:
+ """Test Scrape sensor fails on no data."""
+ config = {"sensor": return_config(select=".current-version h1", name="HA version")}
+
+ mocker = MockRestData("test_scrape_sensor_no_data")
+ with patch(
+ "homeassistant.components.scrape.sensor.RestData",
+ return_value=mocker,
+ ):
+ assert await async_setup_component(hass, "sensor", config)
+ await hass.async_block_till_done()
+
+ state = hass.states.get("sensor.ha_version")
+ assert state is None
+
+
+async def test_scrape_sensor_no_data_refresh(hass: HomeAssistant) -> None:
+ """Test Scrape sensor no data on refresh."""
+ config = {"sensor": return_config(select=".current-version h1", name="HA version")}
+
+ mocker = MockRestData("test_scrape_sensor")
+ with patch(
+ "homeassistant.components.scrape.sensor.RestData",
+ return_value=mocker,
+ ):
+ assert await async_setup_component(hass, "sensor", config)
+ await hass.async_block_till_done()
+
+ state = hass.states.get("sensor.ha_version")
+ assert state
+ assert state.state == "Current Version: 2021.12.10"
+
+ mocker.data = None
+ await async_update_entity(hass, "sensor.ha_version")
+
+ assert mocker.data is None
+ assert state is not None
+ assert state.state == "Current Version: 2021.12.10"
+
+
+async def test_scrape_sensor_attribute_and_tag(hass: HomeAssistant) -> None:
+ """Test Scrape sensor with attribute and tag."""
+ config = {
+ "sensor": [
+ return_config(select="div", name="HA class", index=1, attribute="class"),
+ return_config(select="template", name="HA template"),
+ ]
+ }
+
+ mocker = MockRestData("test_scrape_sensor")
+ with patch(
+ "homeassistant.components.scrape.sensor.RestData",
+ return_value=mocker,
+ ):
+ assert await async_setup_component(hass, "sensor", config)
+ await hass.async_block_till_done()
+
+ state = hass.states.get("sensor.ha_class")
+ assert state.state == "['links']"
+ state2 = hass.states.get("sensor.ha_template")
+ assert state2.state == "Trying to get"
+
+
+async def test_scrape_sensor_errors(hass: HomeAssistant) -> None:
+ """Test Scrape sensor handle errors."""
+ config = {
+ "sensor": [
+ return_config(select="div", name="HA class", index=5, attribute="class"),
+ return_config(select="div", name="HA class2", attribute="classes"),
+ ]
+ }
+
+ mocker = MockRestData("test_scrape_sensor")
+ with patch(
+ "homeassistant.components.scrape.sensor.RestData",
+ return_value=mocker,
+ ):
+ assert await async_setup_component(hass, "sensor", config)
+ await hass.async_block_till_done()
+
+ state = hass.states.get("sensor.ha_class")
+ assert state.state == STATE_UNKNOWN
+ state2 = hass.states.get("sensor.ha_class2")
+ assert state2.state == STATE_UNKNOWN