From 096e7cceed1bba5088cbe041c2fcea6fa6e5c35a Mon Sep 17 00:00:00 2001
From: "J. Nick Koston" <nick@cpanel.net>
Date: Sat, 15 Feb 2020 23:10:23 -0600
Subject: [PATCH] Support XML conversion for RESTful sensors (#31809)
* Support XML conversion for RESTful sensors
Many devices continue to use XML for RESTful
APIs. Interfacing with these APIs requires custom
integrations or command line fork()/exec() overhead
which many of these devices can work with as if
they were JSON using xmltojson via this spec:
https://www.xml.com/pub/a/2006/05/31/converting-between-xml-and-json.html
This change implements converting XML output to
JSON via xmltojson so it can work with the existing
rest sensor component. As the attributes that
usually need to be scraped are deeper in the document
support for passing in a template to find the
JSON attributes that have been added. JSON APIs that
do not have their attributes at the top level
can also benefit from this change.
* Auto convert xml, change out the template for jsonpath
* Address review items and potentially unexpected normalize behavior with jsonpath
* Revert "Address review items and potentially unexpected normalize behavior with jsonpath"
This reverts commit fe9e179092f2a664e3668e0c832856bf5c78d262.
* json_dict[0] turned out to be needed
---
homeassistant/components/rest/manifest.json | 2 +-
homeassistant/components/rest/sensor.py | 28 +++
requirements_all.txt | 2 +
requirements_test_all.txt | 2 +
tests/components/rest/test_sensor.py | 214 +++++++++++++++++++-
5 files changed, 236 insertions(+), 12 deletions(-)
diff --git a/homeassistant/components/rest/manifest.json b/homeassistant/components/rest/manifest.json
index 8c8b7f39609..fd7eea12f7e 100644
--- a/homeassistant/components/rest/manifest.json
+++ b/homeassistant/components/rest/manifest.json
@@ -2,7 +2,7 @@
"domain": "rest",
"name": "RESTful",
"documentation": "https://www.home-assistant.io/integrations/rest",
- "requirements": [],
+ "requirements": ["jsonpath==0.82", "xmltodict==0.12.0"],
"dependencies": [],
"codeowners": []
}
diff --git a/homeassistant/components/rest/sensor.py b/homeassistant/components/rest/sensor.py
index 51120cb350c..36fd27c29a5 100644
--- a/homeassistant/components/rest/sensor.py
+++ b/homeassistant/components/rest/sensor.py
@@ -1,10 +1,13 @@
"""Support for RESTful API sensors."""
import json
import logging
+from xml.parsers.expat import ExpatError
+from jsonpath import jsonpath
import requests
from requests.auth import HTTPBasicAuth, HTTPDigestAuth
import voluptuous as vol
+import xmltodict
from homeassistant.components.sensor import DEVICE_CLASSES_SCHEMA, PLATFORM_SCHEMA
from homeassistant.const import (
@@ -38,7 +41,9 @@ DEFAULT_VERIFY_SSL = True
DEFAULT_FORCE_UPDATE = False
DEFAULT_TIMEOUT = 10
+
CONF_JSON_ATTRS = "json_attributes"
+CONF_JSON_ATTRS_PATH = "json_attributes_path"
METHODS = ["POST", "GET"]
PLATFORM_SCHEMA = PLATFORM_SCHEMA.extend(
@@ -57,6 +62,7 @@ PLATFORM_SCHEMA = PLATFORM_SCHEMA.extend(
vol.Optional(CONF_UNIT_OF_MEASUREMENT): cv.string,
vol.Optional(CONF_DEVICE_CLASS): DEVICE_CLASSES_SCHEMA,
vol.Optional(CONF_USERNAME): cv.string,
+ vol.Optional(CONF_JSON_ATTRS_PATH): cv.string,
vol.Optional(CONF_VALUE_TEMPLATE): cv.template,
vol.Optional(CONF_VERIFY_SSL, default=DEFAULT_VERIFY_SSL): cv.boolean,
vol.Optional(CONF_FORCE_UPDATE, default=DEFAULT_FORCE_UPDATE): cv.boolean,
@@ -84,6 +90,7 @@ def setup_platform(hass, config, add_entities, discovery_info=None):
device_class = config.get(CONF_DEVICE_CLASS)
value_template = config.get(CONF_VALUE_TEMPLATE)
json_attrs = config.get(CONF_JSON_ATTRS)
+ json_attrs_path = config.get(CONF_JSON_ATTRS_PATH)
force_update = config.get(CONF_FORCE_UPDATE)
timeout = config.get(CONF_TIMEOUT)
@@ -120,6 +127,7 @@ def setup_platform(hass, config, add_entities, discovery_info=None):
json_attrs,
force_update,
resource_template,
+ json_attrs_path,
)
],
True,
@@ -140,6 +148,7 @@ class RestSensor(Entity):
json_attrs,
force_update,
resource_template,
+ json_attrs_path,
):
"""Initialize the REST sensor."""
self._hass = hass
@@ -153,6 +162,7 @@ class RestSensor(Entity):
self._attributes = None
self._force_update = force_update
self._resource_template = resource_template
+ self._json_attrs_path = json_attrs_path
@property
def name(self):
@@ -191,12 +201,27 @@ class RestSensor(Entity):
self.rest.update()
value = self.rest.data
+ content_type = self.rest.headers.get("content-type")
+
+ if content_type and content_type.startswith("text/xml"):
+ try:
+ value = json.dumps(xmltodict.parse(value))
+ except ExpatError:
+ _LOGGER.warning(
+ "REST xml result could not be parsed and converted to JSON."
+ )
+ _LOGGER.debug("Erroneous XML: %s", value)
if self._json_attrs:
self._attributes = {}
if value:
try:
json_dict = json.loads(value)
+ if self._json_attrs_path is not None:
+ json_dict = jsonpath(json_dict, self._json_attrs_path)
+ # jsonpath will always store the result in json_dict[0]
+ # so the next line happens to work exactly as needed to
+ # find the result
if isinstance(json_dict, list):
json_dict = json_dict[0]
if isinstance(json_dict, dict):
@@ -240,6 +265,7 @@ class RestData:
self._verify_ssl = verify_ssl
self._timeout = timeout
self.data = None
+ self.headers = None
def set_url(self, url):
"""Set url."""
@@ -259,6 +285,8 @@ class RestData:
verify=self._verify_ssl,
)
self.data = response.text
+ self.headers = response.headers
except requests.exceptions.RequestException as ex:
_LOGGER.error("Error fetching data: %s failed with %s", self._resource, ex)
self.data = None
+ self.headers = None
diff --git a/requirements_all.txt b/requirements_all.txt
index e8f6342d301..3a39bca9e8e 100644
--- a/requirements_all.txt
+++ b/requirements_all.txt
@@ -742,6 +742,7 @@ iperf3==0.1.11
# homeassistant.components.route53
ipify==1.0.0
+# homeassistant.components.rest
# homeassistant.components.verisure
jsonpath==0.82
@@ -2097,6 +2098,7 @@ xfinity-gateway==0.0.4
xknx==0.11.2
# homeassistant.components.bluesound
+# homeassistant.components.rest
# homeassistant.components.startca
# homeassistant.components.ted5000
# homeassistant.components.yr
diff --git a/requirements_test_all.txt b/requirements_test_all.txt
index 3e8eca8d3ac..c502887f2c7 100644
--- a/requirements_test_all.txt
+++ b/requirements_test_all.txt
@@ -278,6 +278,7 @@ iaqualink==0.3.1
# homeassistant.components.influxdb
influxdb==5.2.3
+# homeassistant.components.rest
# homeassistant.components.verisure
jsonpath==0.82
@@ -711,6 +712,7 @@ withings-api==2.1.3
wled==0.2.1
# homeassistant.components.bluesound
+# homeassistant.components.rest
# homeassistant.components.startca
# homeassistant.components.ted5000
# homeassistant.components.yr
diff --git a/tests/components/rest/test_sensor.py b/tests/components/rest/test_sensor.py
index 7e03eb0fd41..30eeae9a8e3 100644
--- a/tests/components/rest/test_sensor.py
+++ b/tests/components/rest/test_sensor.py
@@ -6,6 +6,7 @@ import pytest
from pytest import raises
import requests
from requests.exceptions import RequestException, Timeout
+from requests.structures import CaseInsensitiveDict
import requests_mock
import homeassistant.components.rest.sensor as rest
@@ -166,6 +167,33 @@ class TestRestSensorSetup(unittest.TestCase):
)
assert 2 == mock_req.call_count
+ @requests_mock.Mocker()
+ def test_setup_get_xml(self, mock_req):
+ """Test setup with valid configuration."""
+ mock_req.get("http://localhost", status_code=200)
+ with assert_setup_component(1, "sensor"):
+ assert setup_component(
+ self.hass,
+ "sensor",
+ {
+ "sensor": {
+ "platform": "rest",
+ "resource": "http://localhost",
+ "method": "GET",
+ "value_template": "{{ value_json.key }}",
+ "name": "foo",
+ "unit_of_measurement": DATA_MEGABYTES,
+ "verify_ssl": "true",
+ "timeout": 30,
+ "authentication": "basic",
+ "username": "my username",
+ "password": "my password",
+ "headers": {"Accept": "text/xml"},
+ }
+ },
+ )
+ assert 2 == mock_req.call_count
+
class TestRestSensor(unittest.TestCase):
"""Tests for REST sensor platform."""
@@ -178,13 +206,15 @@ class TestRestSensor(unittest.TestCase):
self.rest.update = Mock(
"rest.RestData.update",
side_effect=self.update_side_effect(
- '{ "key": "' + self.initial_state + '" }'
+ '{ "key": "' + self.initial_state + '" }',
+ CaseInsensitiveDict({"Content-Type": "application/json"}),
),
)
self.name = "foo"
self.unit_of_measurement = DATA_MEGABYTES
self.device_class = None
self.value_template = template("{{ value_json.key }}")
+ self.json_attrs_path = None
self.value_template.hass = self.hass
self.force_update = False
self.resource_template = None
@@ -199,15 +229,17 @@ class TestRestSensor(unittest.TestCase):
[],
self.force_update,
self.resource_template,
+ self.json_attrs_path,
)
def tearDown(self):
"""Stop everything that was started."""
self.hass.stop()
- def update_side_effect(self, data):
+ def update_side_effect(self, data, headers):
"""Side effect function for mocking RestData.update()."""
self.rest.data = data
+ self.rest.headers = headers
def test_name(self):
"""Test the name."""
@@ -229,7 +261,8 @@ class TestRestSensor(unittest.TestCase):
def test_update_when_value_is_none(self):
"""Test state gets updated to unknown when sensor returns no data."""
self.rest.update = Mock(
- "rest.RestData.update", side_effect=self.update_side_effect(None)
+ "rest.RestData.update",
+ side_effect=self.update_side_effect(None, CaseInsensitiveDict()),
)
self.sensor.update()
assert self.sensor.state is None
@@ -239,7 +272,10 @@ class TestRestSensor(unittest.TestCase):
"""Test state gets updated when sensor returns a new status."""
self.rest.update = Mock(
"rest.RestData.update",
- side_effect=self.update_side_effect('{ "key": "updated_state" }'),
+ side_effect=self.update_side_effect(
+ '{ "key": "updated_state" }',
+ CaseInsensitiveDict({"Content-Type": "application/json"}),
+ ),
)
self.sensor.update()
assert "updated_state" == self.sensor.state
@@ -248,7 +284,10 @@ class TestRestSensor(unittest.TestCase):
def test_update_with_no_template(self):
"""Test update when there is no value template."""
self.rest.update = Mock(
- "rest.RestData.update", side_effect=self.update_side_effect("plain_state")
+ "rest.RestData.update",
+ side_effect=self.update_side_effect(
+ "plain_state", CaseInsensitiveDict({"Content-Type": "application/json"})
+ ),
)
self.sensor = rest.RestSensor(
self.hass,
@@ -260,6 +299,7 @@ class TestRestSensor(unittest.TestCase):
[],
self.force_update,
self.resource_template,
+ self.json_attrs_path,
)
self.sensor.update()
assert "plain_state" == self.sensor.state
@@ -269,7 +309,10 @@ class TestRestSensor(unittest.TestCase):
"""Test attributes get extracted from a JSON result."""
self.rest.update = Mock(
"rest.RestData.update",
- side_effect=self.update_side_effect('{ "key": "some_json_value" }'),
+ side_effect=self.update_side_effect(
+ '{ "key": "some_json_value" }',
+ CaseInsensitiveDict({"Content-Type": "application/json"}),
+ ),
)
self.sensor = rest.RestSensor(
self.hass,
@@ -281,6 +324,7 @@ class TestRestSensor(unittest.TestCase):
["key"],
self.force_update,
self.resource_template,
+ self.json_attrs_path,
)
self.sensor.update()
assert "some_json_value" == self.sensor.device_state_attributes["key"]
@@ -289,7 +333,10 @@ class TestRestSensor(unittest.TestCase):
"""Test attributes get extracted from a JSON list[0] result."""
self.rest.update = Mock(
"rest.RestData.update",
- side_effect=self.update_side_effect('[{ "key": "another_value" }]'),
+ side_effect=self.update_side_effect(
+ '[{ "key": "another_value" }]',
+ CaseInsensitiveDict({"Content-Type": "application/json"}),
+ ),
)
self.sensor = rest.RestSensor(
self.hass,
@@ -301,6 +348,7 @@ class TestRestSensor(unittest.TestCase):
["key"],
self.force_update,
self.resource_template,
+ self.json_attrs_path,
)
self.sensor.update()
assert "another_value" == self.sensor.device_state_attributes["key"]
@@ -309,7 +357,10 @@ class TestRestSensor(unittest.TestCase):
def test_update_with_json_attrs_no_data(self, mock_logger):
"""Test attributes when no JSON result fetched."""
self.rest.update = Mock(
- "rest.RestData.update", side_effect=self.update_side_effect(None)
+ "rest.RestData.update",
+ side_effect=self.update_side_effect(
+ None, CaseInsensitiveDict({"Content-Type": "application/json"})
+ ),
)
self.sensor = rest.RestSensor(
self.hass,
@@ -321,6 +372,7 @@ class TestRestSensor(unittest.TestCase):
["key"],
self.force_update,
self.resource_template,
+ self.json_attrs_path,
)
self.sensor.update()
assert {} == self.sensor.device_state_attributes
@@ -331,7 +383,10 @@ class TestRestSensor(unittest.TestCase):
"""Test attributes get extracted from a JSON result."""
self.rest.update = Mock(
"rest.RestData.update",
- side_effect=self.update_side_effect('["list", "of", "things"]'),
+ side_effect=self.update_side_effect(
+ '["list", "of", "things"]',
+ CaseInsensitiveDict({"Content-Type": "application/json"}),
+ ),
)
self.sensor = rest.RestSensor(
self.hass,
@@ -343,6 +398,7 @@ class TestRestSensor(unittest.TestCase):
["key"],
self.force_update,
self.resource_template,
+ self.json_attrs_path,
)
self.sensor.update()
assert {} == self.sensor.device_state_attributes
@@ -353,7 +409,10 @@ class TestRestSensor(unittest.TestCase):
"""Test attributes get extracted from a JSON result."""
self.rest.update = Mock(
"rest.RestData.update",
- side_effect=self.update_side_effect("This is text rather than JSON data."),
+ side_effect=self.update_side_effect(
+ "This is text rather than JSON data.",
+ CaseInsensitiveDict({"Content-Type": "text/plain"}),
+ ),
)
self.sensor = rest.RestSensor(
self.hass,
@@ -365,6 +424,7 @@ class TestRestSensor(unittest.TestCase):
["key"],
self.force_update,
self.resource_template,
+ self.json_attrs_path,
)
self.sensor.update()
assert {} == self.sensor.device_state_attributes
@@ -376,7 +436,8 @@ class TestRestSensor(unittest.TestCase):
self.rest.update = Mock(
"rest.RestData.update",
side_effect=self.update_side_effect(
- '{ "key": "json_state_updated_value" }'
+ '{ "key": "json_state_updated_value" }',
+ CaseInsensitiveDict({"Content-Type": "application/json"}),
),
)
self.sensor = rest.RestSensor(
@@ -389,6 +450,7 @@ class TestRestSensor(unittest.TestCase):
["key"],
self.force_update,
self.resource_template,
+ self.json_attrs_path,
)
self.sensor.update()
@@ -397,6 +459,136 @@ class TestRestSensor(unittest.TestCase):
"json_state_updated_value" == self.sensor.device_state_attributes["key"]
), self.force_update
+ def test_update_with_json_attrs_with_json_attrs_path(self):
+ """Test attributes get extracted from a JSON result with a template for the attributes."""
+ json_attrs_path = "$.toplevel.second_level"
+ value_template = template("{{ value_json.toplevel.master_value }}")
+ value_template.hass = self.hass
+
+ self.rest.update = Mock(
+ "rest.RestData.update",
+ side_effect=self.update_side_effect(
+ '{ "toplevel": {"master_value": "master", "second_level": {"some_json_key": "some_json_value", "some_json_key2": "some_json_value2" } } }',
+ CaseInsensitiveDict({"Content-Type": "application/json"}),
+ ),
+ )
+ self.sensor = rest.RestSensor(
+ self.hass,
+ self.rest,
+ self.name,
+ self.unit_of_measurement,
+ self.device_class,
+ value_template,
+ ["some_json_key", "some_json_key2"],
+ self.force_update,
+ self.resource_template,
+ json_attrs_path,
+ )
+
+ self.sensor.update()
+ assert "some_json_value" == self.sensor.device_state_attributes["some_json_key"]
+ assert (
+ "some_json_value2" == self.sensor.device_state_attributes["some_json_key2"]
+ )
+ assert "master" == self.sensor.state
+
+ def test_update_with_xml_convert_json_attrs_with_json_attrs_path(self):
+ """Test attributes get extracted from a JSON result that was converted from XML with a template for the attributes."""
+ json_attrs_path = "$.toplevel.second_level"
+ value_template = template("{{ value_json.toplevel.master_value }}")
+ value_template.hass = self.hass
+
+ self.rest.update = Mock(
+ "rest.RestData.update",
+ side_effect=self.update_side_effect(
+ "<toplevel><master_value>master</master_value><second_level><some_json_key>some_json_value</some_json_key><some_json_key2>some_json_value2</some_json_key2></second_level></toplevel>",
+ CaseInsensitiveDict({"Content-Type": "text/xml+svg"}),
+ ),
+ )
+ self.sensor = rest.RestSensor(
+ self.hass,
+ self.rest,
+ self.name,
+ self.unit_of_measurement,
+ self.device_class,
+ value_template,
+ ["some_json_key", "some_json_key2"],
+ self.force_update,
+ self.resource_template,
+ json_attrs_path,
+ )
+
+ self.sensor.update()
+ assert "some_json_value" == self.sensor.device_state_attributes["some_json_key"]
+ assert (
+ "some_json_value2" == self.sensor.device_state_attributes["some_json_key2"]
+ )
+ assert "master" == self.sensor.state
+
+ def test_update_with_xml_convert_json_attrs_with_jsonattr_template(self):
+ """Test attributes get extracted from a JSON result that was converted from XML."""
+ json_attrs_path = "$.response"
+ value_template = template("{{ value_json.response.bss.wlan }}")
+ value_template.hass = self.hass
+
+ self.rest.update = Mock(
+ "rest.RestData.update",
+ side_effect=self.update_side_effect(
+ '<?xml version="1.0" encoding="utf-8"?><response><scan>0</scan><ver>12556</ver><count>48</count><ssid>alexander</ssid><bss><valid>0</valid><name>0</name><privacy>0</privacy><wlan>bogus</wlan><strength>0</strength></bss><led0>0</led0><led1>0</led1><led2>0</led2><led3>0</led3><led4>0</led4><led5>0</led5><led6>0</led6><led7>0</led7><btn0>up</btn0><btn1>up</btn1><btn2>up</btn2><btn3>up</btn3><pot0>0</pot0><usr0>0</usr0><temp0>0x0XF0x0XF</temp0><time0> 0</time0></response>',
+ CaseInsensitiveDict({"Content-Type": "text/xml"}),
+ ),
+ )
+ self.sensor = rest.RestSensor(
+ self.hass,
+ self.rest,
+ self.name,
+ self.unit_of_measurement,
+ self.device_class,
+ value_template,
+ ["led0", "led1", "temp0", "time0", "ver"],
+ self.force_update,
+ self.resource_template,
+ json_attrs_path,
+ )
+
+ self.sensor.update()
+ assert "0" == self.sensor.device_state_attributes["led0"]
+ assert "0" == self.sensor.device_state_attributes["led1"]
+ assert "0x0XF0x0XF" == self.sensor.device_state_attributes["temp0"]
+ assert "0" == self.sensor.device_state_attributes["time0"]
+ assert "12556" == self.sensor.device_state_attributes["ver"]
+ assert "bogus" == self.sensor.state
+
+ @patch("homeassistant.components.rest.sensor._LOGGER")
+ def test_update_with_xml_convert_bad_xml(self, mock_logger):
+ """Test attributes get extracted from a XML result with bad xml."""
+ value_template = template("{{ value_json.toplevel.master_value }}")
+ value_template.hass = self.hass
+
+ self.rest.update = Mock(
+ "rest.RestData.update",
+ side_effect=self.update_side_effect(
+ "this is not xml", CaseInsensitiveDict({"Content-Type": "text/xml"})
+ ),
+ )
+ self.sensor = rest.RestSensor(
+ self.hass,
+ self.rest,
+ self.name,
+ self.unit_of_measurement,
+ self.device_class,
+ value_template,
+ ["key"],
+ self.force_update,
+ self.resource_template,
+ self.json_attrs_path,
+ )
+
+ self.sensor.update()
+ assert {} == self.sensor.device_state_attributes
+ assert mock_logger.warning.called
+ assert mock_logger.debug.called
+
class TestRestData(unittest.TestCase):
"""Tests for RestData."""
--
GitLab