Stefan/evcc-io
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

eudataact: simplify data point storage (#31204)

Browse source
This commit is contained in:
andig 2026-06-25 18:46:03 +02:00 • committed by GitHub
parent 77d0d11ad2
commit 26ecf69c6c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 104 additions and 100 deletions
Download patch file Download diff file Expand all files Collapse all files

69
vehicle/vw/eudataact/eudataact_test.go
View file

@ -11,9 +11,9 @@ import (
)
// testProvider returns a provider serving the given static data
func testProvider(data map[string]point) *Provider {
func testProvider(data []point) *Provider {
return &Provider{
statusG: func() (map[string]point, error) {
statusG: func() ([]point, error) {
return data, nil
},
}
@ -31,9 +31,9 @@ func TestStatusPlugStates(t *testing.T) {
}
for _, tc := range tc {
data := map[string]point{
FieldPlugState: {Value: tc.plug},
FieldChargingState: {Value: tc.charge},
data := []point{
{Name: FieldPlugState, Value: tc.plug},
{Name: FieldChargingState, Value: tc.charge},
}
p := testProvider(data)
@ -54,7 +54,7 @@ func TestStatusConservationCharging(t *testing.T) {
}
for _, tc := range tc {
data := map[string]point{tc.field: {Value: tc.value}}
data := []point{{Name: tc.field, Value: tc.value}}
p := testProvider(data)
status, err := p.Status()
@ -79,7 +79,7 @@ func TestStatusChargingScenario(t *testing.T) {
}
for _, tc := range tc {
data := map[string]point{FieldChargingScenario: {Value: tc.scenario}}
data := []point{{Name: FieldChargingScenario, Value: tc.scenario}}
p := testProvider(data)
status, err := p.Status()
@ -135,20 +135,20 @@ func TestMerge(t *testing.T) {
t0 := time.Date(2026, 5, 31, 7, 0, 0, 0, time.UTC)
t1 := time.Date(2026, 5, 31, 8, 0, 0, 0, time.UTC)
dst := map[string]point{
FieldSoc: {Value: "70", Timestamp: t1},
FieldOdometer: {Value: "100", Timestamp: t1},
dst := []point{
{Name: FieldSoc, Value: "70", Timestamp: t1},
{Name: FieldOdometer, Value: "100", Timestamp: t1},
}
src := map[string]point{
FieldSoc: {Value: "80", Timestamp: t0}, // newer dataset wins despite older capture
FieldRangeSecondary: {Value: "200", Timestamp: t1}, // new field -> added
src := []point{
{Name: FieldSoc, Value: "80", Timestamp: t0}, // newer dataset wins despite older capture
{Name: FieldRangeSecondary, Value: "200", Timestamp: t1}, // new field -> added
}
merge(dst, src, 1)
dst = merge(dst, src, 1)
assert.Equal(t, "80", dst[FieldSoc].Value, "newer dataset wins, even with an older timestampUtc")
assert.Equal(t, "100", dst[FieldOdometer].Value, "field absent from src is retained")
assert.Equal(t, "200", dst[FieldRangeSecondary].Value, "new field added")
assert.Equal(t, "80", find(dst, FieldSoc).Value, "newer dataset wins, even with an older timestampUtc")
assert.Equal(t, "100", find(dst, FieldOdometer).Value, "field absent from src is retained")
assert.Equal(t, "200", find(dst, FieldRangeSecondary).Value, "new field added")
}
// TestMergeDeliveryOrder guards the case where the portal stamps fresh values
@ -160,7 +160,7 @@ func TestMergeDeliveryOrder(t *testing.T) {
return ts
}
data := map[string]point{}
var data []point
// datasets in delivery order; capture timestamps go backwards as SoC rises
for i, d := range []struct{ value, capture string }{
{"60", "2026-06-13 14:03:37"}, // delivered 14:10
@ -168,35 +168,35 @@ func TestMergeDeliveryOrder(t *testing.T) {
{"66", "2026-06-13 13:03:16"}, // delivered 15:11
{"75", "2026-06-13 13:52:11"}, // delivered 16:54
} {
merge(data, map[string]point{FieldSoc: {Value: d.value, Timestamp: parse(d.capture)}}, uint64(i+1))
data = merge(data, []point{{Name: FieldSoc, Value: d.value, Timestamp: parse(d.capture)}}, uint64(i+1))
}
assert.Equal(t, "75", data[FieldSoc].Value, "the newest delivered SoC wins")
assert.Equal(t, "75", find(data, FieldSoc).Value, "the newest delivered SoC wins")
}
// TestSocFreshestField reproduces issue #30877: a higher-priority SoC field that
// stops being delivered must not shadow a lower-priority field that keeps rising.
func TestSocFreshestField(t *testing.T) {
data := map[string]point{}
var data []point
var seq uint64
deliver := func(fields map[string]point) {
deliver := func(fields []point) {
seq++
merge(data, fields, seq)
data = merge(data, fields, seq)
}
// first datasets carry both SoC fields at 57, the high-priority field winning
deliver(map[string]point{
FieldBatteryStateReportSoc: {Value: "57"},
FieldHvBatteryLevelValue: {Value: "57.0"},
deliver([]point{
{Name: FieldBatteryStateReportSoc, Value: "57"},
{Name: FieldHvBatteryLevelValue, Value: "57.0"},
})
deliver(map[string]point{
FieldBatteryStateReportSoc: {Value: "57"},
FieldHvBatteryLevelValue: {Value: "57.0"},
deliver([]point{
{Name: FieldBatteryStateReportSoc, Value: "57"},
{Name: FieldHvBatteryLevelValue, Value: "57.0"},
})
// later datasets only refresh the fallback field as the car charges
for _, v := range []string{"58.0", "59.0", "61.0"} {
deliver(map[string]point{FieldHvBatteryLevelValue: {Value: v}})
deliver([]point{{Name: FieldHvBatteryLevelValue, Value: v}})
}
soc, err := testProvider(data).Soc()
@ -227,14 +227,15 @@ func TestSocHvBatteryLevelValid(t *testing.T) {
}
// TestPoints guards that a data point with a generic field name ("value") is
// indexed by its unique key while the name stays indexed (and thus logged).
// stored once yet found by both its unique key and its name.
func TestPoints(t *testing.T) {
data := points([]dataPoint{
{Key: KeyRangeID3, DataFieldName: "value", Value: "317"},
{DataFieldName: FieldOdometer, Value: "22164"},
})
assert.Equal(t, "317", data[KeyRangeID3].Value, "ID.3 range indexed by key")
assert.Equal(t, "317", data["value"].Value, "field name remains indexed")
assert.Equal(t, "22164", data[FieldOdometer].Value, "named field indexed by name")
require.Len(t, data, 2, "ID.3 range stored once, not duplicated under key and name")
assert.Equal(t, "317", find(data, KeyRangeID3).Value, "found by key")
assert.Equal(t, "317", find(data, "value").Value, "found by generic name")
assert.Equal(t, "22164", find(data, FieldOdometer).Value, "named field found by name")
}

16
vehicle/vw/eudataact/provider.go
View file

@ -26,7 +26,7 @@ const (
// timestamp plus portalInterval and a latency margin), so the map is updated as
// soon as the portal delivers a new dataset.
type Provider struct {
statusG func() (map[string]point, error)
statusG func() ([]point, error)
}
// NewProvider creates a vehicle api provider
@ -34,8 +34,8 @@ func NewProvider(log *util.Logger, api *API, vin string, cache time.Duration) *P
v := &Provider{}
s := sharedStore(api)
var cached util.Cacheable[map[string]point]
cached = util.ResettableCached(func() (map[string]point, error) {
var cached util.Cacheable[[]point]
cached = util.ResettableCached(func() ([]point, error) {
ts, err := s.update(vin)
if err != nil {
log.ERROR.Println(err)
@ -62,11 +62,11 @@ func resetDelay(ts time.Time, cache time.Duration) time.Duration {
// lookup returns the freshest present value among the given field names (most to
// least authoritative); the highest Seq wins, equal Seq keeps the priority order.
func lookup(data map[string]point, fields ...string) *point {
func lookup(data []point, fields ...string) *point {
var best *point
for _, f := range fields {
if v, ok := data[f]; ok && (best == nil || v.Seq > best.Seq) {
best = new(v)
if p := find(data, f); p != nil && (best == nil || p.Seq > best.Seq) {
best = p
}
}
return best
@ -82,8 +82,8 @@ func (v *Provider) Soc() (float64, error) {
}
// use battery_level_HV.value when its state reports valid
if s, ok := data[FieldHvBatteryLevelState]; ok && s.Value == hvBatteryLevelValid {
if p, ok := data[FieldHvBatteryLevelValue]; ok {
if s := lookup(data, FieldHvBatteryLevelState); s != nil && s.Value == hvBatteryLevelValid {
if p := lookup(data, FieldHvBatteryLevelValue); p != nil {
return strconv.ParseFloat(p.Value, 64)
}
}

38
vehicle/vw/eudataact/store.go
View file

@ -1,7 +1,6 @@
package eudataact
import (
"maps"
"slices"
"sync"
"time"
@ -26,7 +25,7 @@ type store struct {
type vehicleState struct {
mu sync.Mutex // guards the fields below
identifier string
data map[string]point
data []point
after time.Time
seq uint64 // delivery counter, incremented per merged dataset
}
@ -63,7 +62,7 @@ func (s *store) state(vin string) *vehicleState {
v := s.vehicles[vin]
if v == nil {
v = &vehicleState{data: make(map[string]point)}
v = &vehicleState{}
s.vehicles[vin] = v
}
@ -93,10 +92,7 @@ func (s *store) update(vin string) (time.Time, error) {
return time.Time{}, err
}
content, err := contentDatasets(list)
if err != nil {
return time.Time{}, err
}
content := contentDatasets(list)
var newest time.Time
for _, d := range list {
@ -128,7 +124,7 @@ func (s *store) update(vin string) (time.Time, error) {
}
v.seq++
merge(v.data, data, v.seq)
v.data = merge(v.data, data, v.seq)
if !initial {
logData(s.api.log, data)
@ -147,20 +143,19 @@ func (s *store) update(vin string) (time.Time, error) {
}
// snapshot returns a copy of the merged data for vin
func (s *store) snapshot(vin string) map[string]point {
func (s *store) snapshot(vin string) []point {
v := s.state(vin)
v.mu.Lock()
defer v.mu.Unlock()
return maps.Clone(v.data)
return slices.Clone(v.data)
}
// logData logs every field of data at DEBUG level, sorted by field name, with
// its value and own timestamp in local time.
func logData(log *util.Logger, data map[string]point) {
for _, k := range slices.Sorted(maps.Keys(data)) {
p := data[k]
// logData logs every data point at DEBUG level in arrival order, with its value
// and own timestamp in local time.
func logData(log *util.Logger, data []point) {
for _, p := range data {
log.DEBUG.Printf("recv %s %s: %s (%s)", p.Key, p.Name, p.Value, p.Timestamp.Local().Format("2006-01-02 15:04:05"))
}
}
@ -188,11 +183,16 @@ func pending(content []dataset, after time.Time) []dataset {
return res
}
// merge lets src (the newer dataset) win per field and stamps each field with
// merge folds src (the newer dataset) into dst per id, stamping each point with
// seq, the dataset's delivery sequence (timestampUtc is unreliable).
func merge(dst, src map[string]point, seq uint64) {
for k, p := range src {
func merge(dst, src []point, seq uint64) []point {
for _, p := range src {
p.Seq = seq
dst[k] = p
if e := find(dst, p.id()); e != nil {
*e = p
} else {
dst = append(dst, p)
}
}
return dst
}

81
vehicle/vw/eudataact/types.go
View file

@ -94,6 +94,15 @@ type point struct {
Seq uint64
}
// id is the point's deduplication and lookup identity: its unique GUID when
// present, otherwise the (possibly non-unique) field name.
func (p point) id() string {
if p.Key != "" {
return p.Key
}
return p.Name
}
// datasetFile is the JSON document contained in a dataset zip archive
type datasetFile struct {
VIN string `json:"vin"`
@ -137,18 +146,10 @@ const (
// battery_level_HV.value as a trustworthy SoC reading
const hvBatteryLevelValid = "VALID"
// knownKeys lists data point GUIDs that are indexed by their key instead of the
// generic, non-unique DataFieldName they are delivered with
var knownKeys = map[string]struct{}{
KeyRangeID3: {},
}
// contentDatasets returns the datasets that actually carry content, with their
// delivery time parsed into Timestamp and sorted from oldest to newest. The
// contentDatasets returns the content datasets, sorted oldest to newest. The
// portal emits "..._no_content_found.zip" placeholders while the vehicle is
// asleep, which are skipped. An error is returned when a content dataset's
// timestamp cannot be parsed.
func contentDatasets(list []dataset) ([]dataset, error) {
// asleep; those are skipped.
func contentDatasets(list []dataset) []dataset {
content := make([]dataset, 0, len(list))
for _, d := range list {
if strings.HasSuffix(strings.ToLower(d.Name), "_no_content_found.zip") {
@ -162,15 +163,12 @@ func contentDatasets(list []dataset) ([]dataset, error) {
return a.CreatedOn.Compare(b.CreatedOn)
})
return content, nil
return content
}
// parseDataset extracts the inner JSON document from the dataset zip archive and
// decodes it into the dataset's VIN and a map of data points keyed by the dotted
// data field name. On duplicate field names the entry with the newest timestamp
// wins. The VIN is returned so the caller can drop datasets that do not belong
// to the requested vehicle.
func parseDataset(b []byte) (map[string]point, error) {
// decodes it into its data points.
func parseDataset(b []byte) ([]point, error) {
zr, err := zip.NewReader(bytes.NewReader(b), int64(len(b)))
if err != nil {
return nil, err
@ -206,37 +204,42 @@ func parseDataset(b []byte) (map[string]point, error) {
return points(ds.Data), nil
}
// points indexes data points by field name (newest timestamp wins), and known
// data points additionally by their unique key, as their name is not unique.
func points(data []dataPoint) map[string]point {
res := make(map[string]point, len(data))
// points decodes data points, keeping the newest entry per id (see point.id).
func points(data []dataPoint) []point {
var res []point
set := func(name string, p point) {
if name == "" {
return
}
if cur, ok := res[name]; ok && cur.Timestamp.After(p.Timestamp) {
return
}
res[name] = p
}
for _, p := range data {
if p.Value == "" {
for _, dp := range data {
if dp.Value == "" {
continue
}
var ts time.Time
if p.TimestampUtc != nil {
ts = *p.TimestampUtc
if dp.TimestampUtc != nil {
ts = *dp.TimestampUtc
}
p := point{Key: dp.Key, Name: dp.DataFieldName, Value: dp.Value, Timestamp: ts}
if p.id() == "" {
continue
}
pt := point{Key: p.Key, Name: p.DataFieldName, Value: p.Value, Timestamp: ts}
set(p.DataFieldName, pt)
if _, ok := knownKeys[p.Key]; ok {
set(p.Key, pt)
if e := find(res, p.id()); e != nil {
// newest wins; on equal timestamps the later entry wins
if !e.Timestamp.After(p.Timestamp) {
*e = p
}
continue
}
res = append(res, p)
}
return res
}
// find returns the data point identified by id, matched by Key first and Name
// second, or nil if none is present.
func find(data []point, id string) *point {
if i := slices.IndexFunc(data, func(p point) bool { return p.Key == id || p.Name == id }); i >= 0 {
return &data[i]
}
return nil
}