Add encoding configuration setting to REST and Scape (#90254)

* Create new config parameter for default character encoding if no character encoding is declared

* Changes suggested by gjohansson-ST

* Added config flow for scape

* Removed "character"

* Change to create_async_httpx_client

* Remove CONF_ENCODING from Scrape SENSOR_SCHEMA

* Debug scrape test

homeassistant/components/rest/__init__.py

@@ -41,7 +41,15 @@ from homeassistant.helpers.reload import (
 from homeassistant.helpers.typing import ConfigType, DiscoveryInfoType
 from homeassistant.helpers.update_coordinator import DataUpdateCoordinator
 
-from .const import COORDINATOR, DOMAIN, PLATFORM_IDX, REST, REST_DATA, REST_IDX
+from .const import (
+    CONF_ENCODING,
+    COORDINATOR,
+    DOMAIN,
+    PLATFORM_IDX,
+    REST,
+    REST_DATA,
+    REST_IDX,
+)
 from .data import RestData
 from .schema import CONFIG_SCHEMA, RESOURCE_SCHEMA  # noqa: F401
 
@@ -182,7 +190,7 @@ def create_rest_data_from_config(hass: HomeAssistant, config: ConfigType) -> Res
     headers: dict[str, str] | None = config.get(CONF_HEADERS)
     params: dict[str, str] | None = config.get(CONF_PARAMS)
     timeout: int = config[CONF_TIMEOUT]
-
+    encoding: str = config[CONF_ENCODING]
     if resource_template is not None:
         resource_template.hass = hass
         resource = resource_template.async_render(parse_result=False)
@@ -201,5 +209,14 @@ def create_rest_data_from_config(hass: HomeAssistant, config: ConfigType) -> Res
             auth = (username, password)
 
     return RestData(
-        hass, method, resource, auth, headers, params, payload, verify_ssl, timeout
+        hass,
+        method,
+        resource,
+        encoding,
+        auth,
+        headers,
+        params,
+        payload,
+        verify_ssl,
+        timeout,
     )

homeassistant/components/rest/const.py

@@ -5,6 +5,8 @@ DOMAIN = "rest"
 DEFAULT_METHOD = "GET"
 DEFAULT_VERIFY_SSL = True
 DEFAULT_FORCE_UPDATE = False
+DEFAULT_ENCODING = "UTF-8"
+CONF_ENCODING = "encoding"
 
 DEFAULT_BINARY_SENSOR_NAME = "REST Binary Sensor"
 DEFAULT_SENSOR_NAME = "REST Sensor"

homeassistant/components/rest/data.py

@@ -7,7 +7,7 @@ import httpx
 
 from homeassistant.core import HomeAssistant
 from homeassistant.helpers import template
-from homeassistant.helpers.httpx_client import get_async_client
+from homeassistant.helpers.httpx_client import create_async_httpx_client
 
 DEFAULT_TIMEOUT = 10
 
@@ -22,6 +22,7 @@ class RestData:
         hass: HomeAssistant,
         method: str,
         resource: str,
+        encoding: str,
         auth: httpx.DigestAuth | tuple[str, str] | None,
         headers: dict[str, str] | None,
         params: dict[str, str] | None,
@@ -33,6 +34,7 @@ class RestData:
         self._hass = hass
         self._method = method
         self._resource = resource
+        self._encoding = encoding
         self._auth = auth
         self._headers = headers
         self._params = params
@@ -51,8 +53,8 @@ class RestData:
     async def async_update(self, log_errors: bool = True) -> None:
         """Get the latest data from REST service with provided method."""
         if not self._async_client:
-            self._async_client = get_async_client(
-                self._hass, verify_ssl=self._verify_ssl
+            self._async_client = create_async_httpx_client(
+                self._hass, verify_ssl=self._verify_ssl, default_encoding=self._encoding
             )
 
         rendered_headers = template.render_complex(self._headers, parse_result=False)

homeassistant/components/rest/schema.py

@@ -33,8 +33,10 @@ from homeassistant.helpers.template_entity import (
 )
 
 from .const import (
+    CONF_ENCODING,
     CONF_JSON_ATTRS,
     CONF_JSON_ATTRS_PATH,
+    DEFAULT_ENCODING,
     DEFAULT_FORCE_UPDATE,
     DEFAULT_METHOD,
     DEFAULT_VERIFY_SSL,
@@ -57,6 +59,7 @@ RESOURCE_SCHEMA = {
     vol.Optional(CONF_PAYLOAD): cv.string,
     vol.Optional(CONF_VERIFY_SSL, default=DEFAULT_VERIFY_SSL): cv.boolean,
     vol.Optional(CONF_TIMEOUT, default=DEFAULT_TIMEOUT): cv.positive_int,
+    vol.Optional(CONF_ENCODING, default=DEFAULT_ENCODING): cv.string,
 }
 
 SENSOR_SCHEMA = {

homeassistant/components/scrape/config_flow.py

@@ -60,7 +60,15 @@ from homeassistant.helpers.selector import (
 )
 
 from . import COMBINED_SCHEMA
-from .const import CONF_INDEX, CONF_SELECT, DEFAULT_NAME, DEFAULT_VERIFY_SSL, DOMAIN
+from .const import (
+    CONF_ENCODING,
+    CONF_INDEX,
+    CONF_SELECT,
+    DEFAULT_ENCODING,
+    DEFAULT_NAME,
+    DEFAULT_VERIFY_SSL,
+    DOMAIN,
+)
 
 RESOURCE_SETUP = {
     vol.Required(CONF_RESOURCE): TextSelector(
@@ -84,6 +92,7 @@ RESOURCE_SETUP = {
     vol.Optional(CONF_TIMEOUT, default=DEFAULT_TIMEOUT): NumberSelector(
         NumberSelectorConfig(min=0, step=1, mode=NumberSelectorMode.BOX)
     ),
+    vol.Optional(CONF_ENCODING, default=DEFAULT_ENCODING): TextSelector(),
 }
 
 SENSOR_SETUP = {

homeassistant/components/scrape/const.py

@@ -6,11 +6,13 @@ from datetime import timedelta
 from homeassistant.const import Platform
 
 DOMAIN = "scrape"
+DEFAULT_ENCODING = "UTF-8"
 DEFAULT_NAME = "Web scrape"
 DEFAULT_VERIFY_SSL = True
 DEFAULT_SCAN_INTERVAL = timedelta(minutes=10)
 
 PLATFORMS = [Platform.SENSOR]
 
+CONF_ENCODING = "encoding"
 CONF_SELECT = "select"
 CONF_INDEX = "index"

homeassistant/components/scrape/strings.json

@@ -16,14 +16,16 @@
           "password": "[%key:common::config_flow::data::password%]",
           "headers": "Headers",
           "method": "Method",
-          "timeout": "Timeout"
+          "timeout": "Timeout",
+          "encoding": "Character encoding"
         },
         "data_description": {
           "resource": "The URL to the website that contains the value",
           "authentication": "Type of the HTTP authentication. Either basic or digest",
           "verify_ssl": "Enables/disables verification of SSL/TLS certificate, for example if it is self-signed",
           "headers": "Headers to use for the web request",
-          "timeout": "Timeout for connection to website"
+          "timeout": "Timeout for connection to website",
+          "encoding": "Character encoding to use. Defaults to UTF-8"
         }
       },
       "sensor": {
@@ -110,14 +112,16 @@
           "password": "[%key:component::scrape::config::step::user::data::password%]",
           "headers": "[%key:component::scrape::config::step::user::data::headers%]",
           "verify_ssl": "[%key:component::scrape::config::step::user::data::verify_ssl%]",
-          "timeout": "[%key:component::scrape::config::step::user::data::timeout%]"
+          "timeout": "[%key:component::scrape::config::step::user::data::timeout%]",
+          "encoding": "[%key:component::scrape::config::step::user::data::encoding%]"
         },
         "data_description": {
           "resource": "[%key:component::scrape::config::step::user::data_description::resource%]",
           "authentication": "[%key:component::scrape::config::step::user::data_description::authentication%]",
           "headers": "[%key:component::scrape::config::step::user::data_description::headers%]",
           "verify_ssl": "[%key:component::scrape::config::step::user::data_description::verify_ssl%]",
-          "timeout": "[%key:component::scrape::config::step::user::data_description::timeout%]"
+          "timeout": "[%key:component::scrape::config::step::user::data_description::timeout%]",
+          "encoding": "[%key:component::scrape::config::step::user::data_description::encoding%]"
         }
       }
     }

tests/components/scrape/conftest.py

@@ -9,7 +9,13 @@ import pytest
 
 from homeassistant.components.rest.data import DEFAULT_TIMEOUT
 from homeassistant.components.rest.schema import DEFAULT_METHOD, DEFAULT_VERIFY_SSL
-from homeassistant.components.scrape.const import CONF_INDEX, CONF_SELECT, DOMAIN
+from homeassistant.components.scrape.const import (
+    CONF_ENCODING,
+    CONF_INDEX,
+    CONF_SELECT,
+    DEFAULT_ENCODING,
+    DOMAIN,
+)
 from homeassistant.config_entries import SOURCE_USER
 from homeassistant.const import (
     CONF_METHOD,
@@ -38,6 +44,7 @@ async def get_config_to_integration_load() -> dict[str, Any]:
         CONF_METHOD: DEFAULT_METHOD,
         CONF_VERIFY_SSL: DEFAULT_VERIFY_SSL,
         CONF_TIMEOUT: DEFAULT_TIMEOUT,
+        CONF_ENCODING: DEFAULT_ENCODING,
         "sensor": [
             {
                 CONF_NAME: "Current version",

tests/components/scrape/test_config_flow.py

@@ -9,8 +9,10 @@ from homeassistant.components.rest.data import DEFAULT_TIMEOUT
 from homeassistant.components.rest.schema import DEFAULT_METHOD
 from homeassistant.components.scrape import DOMAIN
 from homeassistant.components.scrape.const import (
+    CONF_ENCODING,
     CONF_INDEX,
     CONF_SELECT,
+    DEFAULT_ENCODING,
     DEFAULT_VERIFY_SSL,
 )
 from homeassistant.const import (
@@ -75,6 +77,7 @@ async def test_form(hass: HomeAssistant, get_data: MockRestData) -> None:
         CONF_METHOD: "GET",
         CONF_VERIFY_SSL: True,
         CONF_TIMEOUT: 10.0,
+        CONF_ENCODING: "UTF-8",
         "sensor": [
             {
                 CONF_NAME: "Current version",
@@ -165,6 +168,7 @@ async def test_flow_fails(hass: HomeAssistant, get_data: MockRestData) -> None:
         CONF_METHOD: "GET",
         CONF_VERIFY_SSL: True,
         CONF_TIMEOUT: 10.0,
+        CONF_ENCODING: "UTF-8",
         "sensor": [
             {
                 CONF_NAME: "Current version",
@@ -206,6 +210,7 @@ async def test_options_resource_flow(
                 CONF_METHOD: DEFAULT_METHOD,
                 CONF_VERIFY_SSL: DEFAULT_VERIFY_SSL,
                 CONF_TIMEOUT: DEFAULT_TIMEOUT,
+                CONF_ENCODING: DEFAULT_ENCODING,
                 CONF_USERNAME: "secret_username",
                 CONF_PASSWORD: "secret_password",
             },
@@ -218,6 +223,7 @@ async def test_options_resource_flow(
         CONF_METHOD: "GET",
         CONF_VERIFY_SSL: True,
         CONF_TIMEOUT: 10.0,
+        CONF_ENCODING: "UTF-8",
         CONF_USERNAME: "secret_username",
         CONF_PASSWORD: "secret_password",
         "sensor": [
@@ -282,6 +288,7 @@ async def test_options_add_remove_sensor_flow(
         CONF_METHOD: "GET",
         CONF_VERIFY_SSL: True,
         CONF_TIMEOUT: 10,
+        CONF_ENCODING: "UTF-8",
         "sensor": [
             {
                 CONF_NAME: "Current version",
@@ -341,6 +348,7 @@ async def test_options_add_remove_sensor_flow(
         CONF_METHOD: "GET",
         CONF_VERIFY_SSL: True,
         CONF_TIMEOUT: 10,
+        CONF_ENCODING: "UTF-8",
         "sensor": [
             {
                 CONF_NAME: "Template",
@@ -407,6 +415,7 @@ async def test_options_edit_sensor_flow(
         CONF_METHOD: "GET",
         CONF_VERIFY_SSL: True,
         CONF_TIMEOUT: 10,
+        CONF_ENCODING: "UTF-8",
         "sensor": [
             {
                 CONF_NAME: "Current version",