chore: refactor plugin structure (#18335)

plugin/pipeline/pipeline.go

@@ -0,0 +1,224 @@
+package pipeline
+
+import (
+	"bytes"
+	"encoding/hex"
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+
+	xj "github.com/basgys/goxml2json"
+	"github.com/cenkalti/backoff/v4"
+	"github.com/evcc-io/evcc/util"
+	"github.com/evcc-io/evcc/util/jq"
+	"github.com/itchyny/gojq"
+	"github.com/volkszaehler/mbmd/meters/rs485"
+)
+
+type Pipeline struct {
+	log        *util.Logger
+	re         *regexp.Regexp
+	jq         *gojq.Query
+	allowEmpty bool
+	quote      bool
+	dflt       string
+	unpack     string
+	decode     string
+}
+
+type Settings struct {
+	AllowEmpty bool
+	Quote      bool
+	Regex      string
+	Default    string
+	Jq         string
+	Unpack     string
+	Decode     string
+}
+
+func New(log *util.Logger, cc Settings) (*Pipeline, error) {
+	p := &Pipeline{
+		log:        log,
+		allowEmpty: cc.AllowEmpty,
+		quote:      cc.Quote,
+	}
+
+	var err error
+	if err == nil && cc.Regex != "" {
+		_, err = p.WithRegex(cc.Regex, cc.Default)
+	}
+
+	if err == nil && cc.Jq != "" {
+		_, err = p.WithJq(cc.Jq)
+	}
+
+	if err == nil && cc.Unpack != "" {
+		_, err = p.WithUnpack(cc.Unpack)
+	}
+
+	if err == nil && cc.Decode != "" {
+		_, err = p.WithDecode(cc.Decode)
+	}
+
+	return p, err
+}
+
+// WithRegex adds a regex query applied to the mqtt listener payload
+func (p *Pipeline) WithRegex(regex, dflt string) (*Pipeline, error) {
+	re, err := regexp.Compile(regex)
+	if err != nil {
+		return nil, fmt.Errorf("invalid regex '%s': %w", re, err)
+	}
+
+	p.re = re
+	p.dflt = dflt
+
+	return p, nil
+}
+
+// WithJq adds a jq query applied to the mqtt listener payload
+func (p *Pipeline) WithJq(jq string) (*Pipeline, error) {
+	op, err := gojq.Parse(jq)
+	if err != nil {
+		return nil, fmt.Errorf("invalid jq query '%s': %w", jq, err)
+	}
+
+	p.jq = op
+
+	return p, nil
+}
+
+// WithUnpack adds data unpacking
+func (p *Pipeline) WithUnpack(unpack string) (*Pipeline, error) {
+	p.unpack = strings.ToLower(unpack)
+
+	return p, nil
+}
+
+// WithDecode adds data decoding
+func (p *Pipeline) WithDecode(decode string) (*Pipeline, error) {
+	p.decode = strings.ToLower(decode)
+
+	return p, nil
+}
+
+// transform XML into JSON with attribute names getting 'attr' prefix
+func (p *Pipeline) transformXML(value []byte) []byte {
+	value = bytes.TrimSpace(value)
+
+	// only do a simple check, as some devices e.g. Kostal Piko MP plus don't seem to send proper XML
+	if !bytes.HasPrefix(value, []byte("<?xml")) {
+		return value
+	}
+
+	in := bytes.NewReader(value)
+
+	// Decode XML document
+	root := new(xj.Node)
+	if err := xj.NewDecoder(in).DecodeWithCustomPrefixes(root, "", "attr"); err != nil {
+		return value
+	}
+
+	// Then encode it in JSON
+	out := new(bytes.Buffer)
+	if err := xj.NewEncoder(out).Encode(root); err != nil {
+		return value
+	}
+
+	if p.log != nil {
+		p.log.TRACE.Println(out.String())
+	}
+
+	return out.Bytes()
+}
+
+func (p *Pipeline) unpackValue(value []byte) (string, error) {
+	switch p.unpack {
+	case "hex":
+		b, err := hex.DecodeString(string(value))
+		if err != nil {
+			return "", err
+		}
+		return string(b), nil
+	}
+
+	return "", fmt.Errorf("invalid unpack: %s", p.unpack)
+}
+
+// decode a hex string to a proper value
+// TODO reuse similar code from Modbus
+func (p *Pipeline) decodeValue(value []byte) (float64, error) {
+	switch p.decode {
+	case "float32", "ieee754":
+		return rs485.RTUIeee754ToFloat64(value), nil
+	case "float32s", "ieee754s":
+		return rs485.RTUIeee754ToFloat64Swapped(value), nil
+	case "float64":
+		return rs485.RTUUint64ToFloat64(value), nil
+	case "uint16":
+		return rs485.RTUUint16ToFloat64(value), nil
+	case "uint32":
+		return rs485.RTUUint32ToFloat64(value), nil
+	case "uint32s":
+		return rs485.RTUUint32ToFloat64Swapped(value), nil
+	case "uint64":
+		return rs485.RTUUint64ToFloat64(value), nil
+	case "int16":
+		return rs485.RTUInt16ToFloat64(value), nil
+	case "int32":
+		return rs485.RTUInt32ToFloat64(value), nil
+	case "int32s":
+		return rs485.RTUInt32ToFloat64Swapped(value), nil
+	}
+
+	return 0, fmt.Errorf("invalid decoding: %s", p.decode)
+}
+
+func (p *Pipeline) Process(in []byte) ([]byte, error) {
+	if p.allowEmpty && len(bytes.TrimSpace(in)) == 0 {
+		return nil, nil
+	}
+
+	b := p.transformXML(in)
+
+	if p.re != nil {
+		m := p.re.FindSubmatch(b)
+		if len(m) == 1 {
+			b = m[0] // full match
+		} else if len(m) > 1 {
+			b = m[1] // first submatch
+		} else if len(p.dflt) > 0 {
+			return []byte(p.dflt), nil
+		}
+	}
+
+	if p.jq != nil {
+		if p.quote {
+			b = []byte(fmt.Sprintf("%q", string(b)))
+		}
+		v, err := jq.Query(p.jq, b)
+		if err != nil {
+			return b, backoff.Permanent(err)
+		}
+		b = []byte(fmt.Sprintf("%v", v))
+	}
+
+	if p.unpack != "" {
+		v, err := p.unpackValue(b)
+		if err != nil {
+			return b, err
+		}
+		b = []byte(fmt.Sprintf("%v", v))
+	}
+
+	if p.decode != "" {
+		v, err := p.decodeValue(b)
+		if err != nil {
+			return b, err
+		}
+		b = []byte(strconv.FormatFloat(v, 'f', -1, 64))
+	}
+
+	return b, nil
+}

plugin/pipeline/pipeline_test.go

@@ -0,0 +1,51 @@
+package pipeline
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestRegex(t *testing.T) {
+	for _, re := range []string{`([0-9.]+)`, `[0-9.]+`} {
+		p, err := new(Pipeline).WithRegex(re, "")
+		require.NoError(t, err)
+
+		res, err := p.Process([]byte("12.3W"))
+		require.NoError(t, err)
+
+		require.Equal(t, []byte("12.3"), res)
+	}
+}
+
+func TestRegexDefault(t *testing.T) {
+	p, err := new(Pipeline).WithRegex(`\d+`, "123")
+	require.NoError(t, err)
+
+	res, err := p.Process([]byte("xxx"))
+	require.NoError(t, err)
+
+	require.Equal(t, []byte("123"), res)
+}
+
+func TestJq(t *testing.T) {
+	for _, uuid := range []string{"a8232ee0-a4ab-11ec-8d36-211f6b082dc8", "08232ee0-a4ab-11ec-8d36-211f6b082dc8"} {
+		p, err := new(Pipeline).WithJq(fmt.Sprintf(`.data[] | select(.uuid=="%s") | .tuples[0][1]`, uuid))
+		require.NoError(t, err)
+
+		res, err := p.Process([]byte(`
+			{
+				"data": [
+					{
+						"uuid": "` + uuid + `",
+						"tuples": [[1,2,3]]
+					}
+				]
+			}
+		`))
+		require.NoError(t, err)
+
+		require.Equal(t, []byte("2"), res)
+	}
+}