Add xml transformation to HTTP provider with Kostal Piko MP Plus example (#2120)

- Added automatic transformation of XML responses into JSON, so the data can be used with jq
- Add Kostal Piko MP Plus template that uses the above addition

go.mod

@@ -11,7 +11,9 @@ require (
 	github.com/asaskevich/EventBus v0.0.0-20200907212545-49d423059eef
 	github.com/avast/retry-go/v3 v3.1.1
 	github.com/aws/aws-sdk-go v1.42.23
+	github.com/basgys/goxml2json v1.1.0
 	github.com/benbjohnson/clock v1.3.0
+	github.com/bitly/go-simplejson v0.5.0 // indirect
 	github.com/bogosj/tesla v1.0.2
 	github.com/cjrd/allocate v0.0.0-20191115010018-022b87fe59fc
 	github.com/cloudfoundry/jibber_jabber v0.0.0-20151120183258-bcc4c8345a21

go.sum

@@ -94,6 +94,8 @@ github.com/avast/retry-go/v3 v3.1.1 h1:49Scxf4v8PmiQ/nY0aY3p0hDueqSmc7++cBbtiDGu
 github.com/avast/retry-go/v3 v3.1.1/go.mod h1:6cXRK369RpzFL3UQGqIUp9Q7GDrams+KsYWrfNA1/nQ=
 github.com/aws/aws-sdk-go v1.42.23 h1:V0V5hqMEyVelgpu1e4gMPVCJ+KhmscdNxP/NWP1iCOA=
 github.com/aws/aws-sdk-go v1.42.23/go.mod h1:gyRszuZ/icHmHAVE4gc/r+cfCmhA1AD+vqfWbgI+eHs=
+github.com/basgys/goxml2json v1.1.0 h1:4ln5i4rseYfXNd86lGEB+Vi652IsIXIvggKM/BhUKVw=
+github.com/basgys/goxml2json v1.1.0/go.mod h1:wH7a5Np/Q4QoECFIU8zTQlZwZkrilY0itPfecMw41Dw=
 github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A=
 github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
@@ -101,6 +103,8 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
+github.com/bitly/go-simplejson v0.5.0 h1:6IH+V8/tVMab511d5bn4M7EwGXZf9Hj6i2xSwkNEM+Y=
+github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA=
 github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM=
 github.com/bogosj/tesla v1.0.2 h1:6i1RCjywaAt06ypeUWDIm/7vh609tZs/ubIInmVfyHA=
 github.com/bogosj/tesla v1.0.2/go.mod h1:LPbTjUSyUTPhTyi6fpybXnv2t/Hv0V8FX6+hyEIpL98=

provider/http.go

@@ -1,6 +1,7 @@
 package provider
 
 import (
+	"bytes"
 	"encoding/hex"
 	"fmt"
 	"io"
@@ -10,6 +11,7 @@ import (
 	"strings"
 	"time"
 
+	xj "github.com/basgys/goxml2json"
 	"github.com/evcc-io/evcc/provider/javascript"
 	"github.com/evcc-io/evcc/util"
 	"github.com/evcc-io/evcc/util/jq"
@@ -235,6 +237,30 @@ func (p *HTTP) request(body ...string) ([]byte, error) {
 	return p.val, p.err
 }
 
+// transform XML into JSON with attribute names getting 'attr' prefix
+func (p *HTTP) transformXML(value []byte) []byte {
+	// only do a simple check, as some devices e.g. Kostal Piko MP plus don't seem to send proper XML
+	if !bytes.HasPrefix(value, []byte("<")) {
+		return value
+	}
+
+	xmlReader := bytes.NewReader(value)
+
+	// Decode XML document
+	root := new(xj.Node)
+	if err := xj.NewDecoder(xmlReader).DecodeWithCustomPrefixes(root, "", "attr"); err != nil {
+		return value
+	}
+
+	// Then encode it in JSON
+	json := new(bytes.Buffer)
+	if err := xj.NewEncoder(json).Encode(root); err != nil {
+		return value
+	}
+
+	return json.Bytes()
+}
+
 func (p *HTTP) unpackValue(value []byte) (string, error) {
 	switch p.unpack {
 	case "hex":
@@ -314,6 +340,8 @@ func (p *HTTP) StringGetter() func() (string, error) {
 			return string(b), err
 		}
 
+		b = p.transformXML(b)
+
 		if p.re != nil {
 			m := p.re.FindSubmatch(b)
 			if len(m) > 1 {

templates/definition/meter/kostal-piko-mp-plus.yaml

@@ -0,0 +1,19 @@
+template: kostal-piko-mp-plus
+description: Kostal Piko MP Plus
+guidedsetup:
+  enable: true
+params:
+- name: usage
+  choice: [ "pv" ]
+- name: host
+  example: 192.0.2.2
+  required: true
+render: |
+  type: custom
+  power:
+  {{- if eq .usage "pv" }}
+    # PV
+    source: http
+    uri: http://{{ .host }}/measurements.xml
+    jq: .root.Device.Measurements.Measurement[] | select(.attrType == "AC_Power") | .attrValue | tonumber
+  {{- end -}}

templates/docs/meter/kostal-piko-mp-plus-pv.yaml

@@ -0,0 +1,5 @@
+type: template
+template: kostal-piko-mp-plus
+description: Kostal Piko MP Plus
+usage: pv 
+host: 192.0.2.2