From f60eed3ff6b09ccef62be7a15f34b6b1d4f7dff3 Mon Sep 17 00:00:00 2001 From: Max Gramser Date: Mon, 14 Sep 2026 21:06:45 +0200 Subject: [PATCH] Make daily traffic scale to dozens of screens: incremental sync, ping keepalive, one action per message (0.2.39) App 0.2.39 / firmware 0.2.33, items 1, 2, 4, 5 and 6 of the 30-screen scaling check. - Only the tiles (and top bar) of changed entities are rebuilt; the screen list is cached until the registry or a screen diagnostic changes; the SSE stream follows. - Sensor history comes from one recorder/statistics_during_period call per window in a background task, with the REST history as fallback per sensor without statistics. - Firmware 0.2.33 answers a keepalive ping with its layout revision; a mismatch or a restart asks for the layout again, everything is repeated hourly as a safety net. - The screen_message API action takes one whole message; the text inbox chunks stay as fallback for older firmware. A repeated unchanged layout keeps the inbox state. - Quiet diagnostics: Opgestart (uptime as timestamp) replaces Uptime, heap every 5 min, setting numbers publish only on change. Co-Authored-By: Claude Fable 5.1 --- README.md | 10 + components/smart_display/runtime_tiles.h | 20 +- docs/RELEASING.md | 51 +++ docs/TEST_RESULTS_0239.md | 92 +++++ guition-4848s040.yaml | 34 +- home-like-2432s028.yaml | 34 +- packages/cyd.yaml | 34 +- packages/guition.yaml | 34 +- screen_manager/CHANGELOG.md | 11 + screen_manager/app/core.py | 71 +++- screen_manager/app/server.py | 407 ++++++++++++++++++----- screen_manager/app/updates.py | 4 +- screen_manager/config.yaml | 2 +- tests/test_alerts_reference.py | 2 +- tests/test_header_bar.py | 2 +- tests/test_portal.py | 4 +- tests/test_scaling.py | 323 ++++++++++++++++++ tests/test_updates.py | 2 +- 18 files changed, 990 insertions(+), 147 deletions(-) create mode 100644 docs/TEST_RESULTS_0239.md create mode 100644 tests/test_scaling.py diff --git a/README.md b/README.md index 7b8fa4a..d2a155f 100644 --- a/README.md +++ b/README.md @@ -232,6 +232,16 @@ schermvoorkeuren blijven op het apparaat opgeslagen. Updates vervangen deze gebruikersgegevens niet. Maak wel normale Home Assistant-back-ups en bewaar je apparaatprofielen; een app verwijderen of flashgeheugen wissen is geen update. +**Zuinig, ook met veel schermen (0.2.39 / firmware 0.2.33).** De app stuurt een +scherm alleen de tegel die veranderde, als één actie +(`esphome._screen_message`) in plaats van blokjes in een tekstveld. +Elke twee minuten volgt een kleine ping met de revisie van de indeling; meldt het +scherm dat die niet klopt (na een herstart bijvoorbeeld), dan komt alles opnieuw. +Grafieken van sensortegels komen uit de statistieken van Home Assistant, in één +vraag voor alle schermen. Oudere firmware blijft werken via het tekstveld en de +volledige herhaling per twee minuten. De diagnostische sensor `Uptime` is +vervangen door het tijdstempel `Opgestart`. + Zie [releasegeschiedenis](screen_manager/CHANGELOG.md) en [releases en protocolcompatibiliteit](docs/RELEASING.md). diff --git a/components/smart_display/runtime_tiles.h b/components/smart_display/runtime_tiles.h index 6ad2888..05207e0 100644 --- a/components/smart_display/runtime_tiles.h +++ b/components/smart_display/runtime_tiles.h @@ -53,6 +53,9 @@ inline uint32_t last_received = 0; // Seconds between the manager's full repeats; every layout message declares it (app // 0.2.26+). Older managers get the 120 s that app 0.2.20 introduced. inline uint32_t keepalive_seconds = 120; +// Revision of the layout the manager sent last (app 0.2.39+). A keepalive ping carries the +// manager's revision; a mismatch (a restart, a demo layout) asks for the whole layout again. +inline std::string layout_rev; inline std::function layout_changed, refresh, dismiss, settings_changed; inline esphome::ESPPreferenceObject settings_preference; inline void load_settings() { @@ -177,7 +180,7 @@ inline std::string receive(const std::string &payload) { } } inbox = string(root["inbox"], 160); - bool changed = false, moved = false; + bool changed = false, moved = false, was_configured = model.configured; if (!model.set_layout(entities, string(root["title"], 96), changed, positions, moved)) return false; // Empty pages the user keeps on purpose; absent on older managers. model.pages = root["pages"].is() ? static_cast(std::clamp(root["pages"].as(), 1, MAX_PAGES)) : 1; @@ -197,10 +200,23 @@ inline std::string receive(const std::string &payload) { if (changed) { active_index = -1; for (auto &w : widgets) w.cached_active = -1; if (dismiss) dismiss(); } if (moved) ESP_LOGI("runtime", "tegels verplaatst: pagina's opnieuw ingedeeld"); if (root["keepalive"].is()) keepalive_seconds = root["keepalive"].as(); + layout_rev = string(root["rev"], 16); last_received = esphome::millis(); if (layout_changed) layout_changed(); if (refresh) refresh(); - result = "Indeling ontvangen"; + // A repeat of the same layout keeps the inbox state as it is: no new recorder row. + result = changed || moved || !was_configured ? "Indeling ontvangen" : model.ready() ? "Gesynchroniseerd" : "Tegels laden"; + return true; + } + if (op == "ping") { + // Keepalive without content (app 0.2.39+): only the revision of the layout the manager holds. + if (!root["rev"].is()) return false; + if(!root["keepalive"].isNull() && (!root["keepalive"].is() || + root["keepalive"].as()<5 || root["keepalive"].as()>3600))return false; + if (root["keepalive"].is()) keepalive_seconds = root["keepalive"].as(); + last_received = esphome::millis(); + if (!model.configured || layout_rev != string(root["rev"], 16)) { result = "Indeling opnieuw nodig"; return true; } + result = model.ready() ? "Gesynchroniseerd" : "Tegels laden"; return true; } if (op == "header") { diff --git a/docs/RELEASING.md b/docs/RELEASING.md index 51eb6c4..2bffc66 100644 --- a/docs/RELEASING.md +++ b/docs/RELEASING.md @@ -371,3 +371,54 @@ domeinen in de hele indeling; de manager stuurt zo'n indeling daarom pas na firmware 0.2.14 (`min_firmware`) en bewaart haar ondertussen. Voorspellingen komen via `weather.get_forecasts` met `return_response`; zonder antwoord blijft de weerkaart de gewone kaart. Geen gewijzigde preferences of sleutels. + +### Compatibiliteit 0.2.39 / firmware 0.2.33 + +Opslagversie en tegelprotocol blijven 1; geen nieuwe velden in `screens.json`, geen +gewijzigde preferences of sleutels. Alles is additief en per scherm gekoppeld aan de +gemelde `Schermfirmware`: + +- **Transport.** Firmware 0.2.33 heeft de API-actie `screen_message` (één string + `message`, het complete JSON-bericht, maximaal 4096 bytes). De manager roept + `esphome._screen_message` aan zodra het scherm 0.2.33+ meldt + (`TRANSPORT_MIN_FIRMWARE` in `core.py`, `Manager.transport`); daaronder blijven de + base64-blokjes in de tekstentiteit `Tegelinstellingen`. De actie publiceert het + resultaat op dezelfde inbox-entiteit, dus de status op de beheerpagina blijft gelijk. + Het API-frame van ESPHome op de ESP32 is 32 KiB; `receive()` weigert boven 4096. +- **Keepalive.** Het layoutbericht krijgt `rev` (twaalf hex-tekens, `core.revision` + van het layoutbericht zonder `rev`); oudere firmware negeert het veld. Firmware + 0.2.33 bewaart de revisie en beantwoordt `{v:1, op:"ping", rev, keepalive}` met + `Gesynchroniseerd`/`Tegels laden` (revisie gelijk) of `Indeling opnieuw nodig` + (anders, of na een herstart). Een ping ververst het feed-venster (`last_received`) + net als een layout. De manager pingt elke `KEEPALIVE_SECONDS` (120 s) en herhaalt + alles pas na `FULL_REPEAT_SECONDS` (3600 s), of direct wanneer de inbox een status + uit `RESEND_STATES` meldt (`Klaar voor tegelconfiguratie`, `Indeling opnieuw + nodig`, `Tegels laden`), met een guard van 120 s na de laatste volledige verzending. + Firmware onder 0.2.33 krijgt elke 120 s de volledige herhaling zoals voorheen. + Een herhaald, ongewijzigd layoutbericht antwoordt nu `Gesynchroniseerd` in plaats + van `Indeling ontvangen`; `diagnostics/send_layout.py` accepteert beide. +- **Incrementeel.** `HomeAssistant.dirty` verzamelt de gewijzigde entity-id's; + `Manager.sync_one(..., dirty=...)` bouwt alleen die tegels en, als een van hen in + de bovenbalk staat, de balk. `dirty=None` (eerste ronde, nieuw register, offline + geweest) bouwt alles en stuurt de verschillen; `force=True` stuurt alles. `sent` + bewaart per scherm `{'layout','header','states','rev'}`. `Manager.screens()` + vervangt `inventory()[0]` in de lus en de updater: alleen de ESPHome-schermentiteiten + worden gelezen, gecachet op register-identiteit plus de status van die entiteiten. + `inventory()` (volledig) blijft voor de editor en `save()`. +- **Historie.** `Manager.history_loop` (eigen taak in `main()`) haalt per venster + (1/6/24 uur) één `recorder/statistics_during_period` op (`hour` voor 24 uur, + `5minute` daaronder, `types: [mean, state]`; `mean`, anders `state`), valt per + sensor zonder rijen terug op `GET /history/period`, en markeert gewijzigde + entiteiten dirty. Het `history`-veld in het statusbericht is ongewijzigd (24 + waarden, `None` tot de eerste). Een sensortegel gaat de eerste keer zonder + `history` de deur uit en krijgt hem seconden later; firmware toont dan de grafiek. +- **Diagnostiek (firmware).** De sensor `Uptime` (seconden, elke 15 s) is weg; + ervoor in de plaats `Opgestart` (`platform: uptime`, `type: timestamp`, device + class timestamp, één waarde per opstart; `time:` is al aanwezig). Bewust een nieuwe + entiteit: het entity-register van Home Assistant houdt bij de oude de eenheid `s` + vast en weigert dan een tijdstempel ("has a unit of measurement ... non-numeric + device class: timestamp", gezien op 2026-09-14). De ESPHome-integratie verwijdert + de oude `sensor._uptime` zelf zodra het scherm met de nieuwe firmware + verbindt. Verder `debug: update_interval` 300 s en de vier template-numbers op + `update_interval: never`, gepubliceerd vanuit `apply_screen_settings` als de waarde + afwijkt van hun laatste state (dus ook eenmaal na opstarten). diff --git a/docs/TEST_RESULTS_0239.md b/docs/TEST_RESULTS_0239.md new file mode 100644 index 0000000..3e0d633 --- /dev/null +++ b/docs/TEST_RESULTS_0239.md @@ -0,0 +1,92 @@ +# Testresultaten app 0.2.39 / firmware 0.2.33 (2026-09-14) + +Dagelijkse efficiëntie met veel schermen: aanpassingen 1, 2, 4, 5 en 6 uit de +schaalcheck (incrementeel doorrekenen, historie gebundeld op de achtergrond, keepalive +als ping, één API-actie per bericht, rustige diagnostiek). Zie CHANGELOG 0.2.39 en +RELEASING "Compatibiliteit 0.2.39". + +## Wat veranderde + +- App: `HomeAssistant.dirty` + `Manager.sync_one(..., dirty)` bouwt alleen gewijzigde + tegels en de bovenbalk als die er een toont; `Manager.screens()` (gecachte + schermlijst uit alleen de ESPHome-schermentiteiten) vervangt `inventory()[0]` in de + lus, de SSE-stream en de updater; `history_loop` + `refresh_histories` (één + `recorder/statistics_during_period` per venster, REST-terugval, dirty-markering); + `ping()` elke 120 s met `rev`, volledige herhaling per uur of bij een inbox-status + uit `RESEND_STATES`; `HomeAssistant.send(..., action)` roept + `esphome._screen_message` aan voor firmware 0.2.33+. +- Firmware: `runtime_tiles::layout_rev`, `op: ping`, herhaalde ongewijzigde layout + antwoordt `Gesynchroniseerd`; API-actie `screen_message`; `Opgestart` (uptime als + tijdstempel) vervangt `Uptime`; `debug` 300 s; instellingsgetallen publiceren alleen + bij verandering vanuit `apply_screen_settings`. + +## Geautomatiseerd + +- Python (`.venv-portal`): 130 tests OK, waarvan 9 nieuw in `test_scaling.py` + (encode/revisie/actienaam, bucketing, alleen dirty tegels herbouwd en verstuurd, + bovenbalk volgt eigen entiteit, `dirty=None` stuurt verschillen, `force` alles, + ping-inhoud, oude firmware zonder actie en met volledige keepalive, de echte + `run()`-lus: eerste ronde, stille wake, ping na 120 s, hersturen na + `Indeling opnieuw nodig` met guard, uurlijkse herhaling, één dirty tegel; + schermcache alleen opnieuw bij register of schermdiagnose en met kopieën; + statistiekbundel per venster, REST-terugval, dirty alleen bij gewijzigde waarden, + cache opgeruimd; `HomeAssistant.statistics` met ms-tijdstempels, `mean`/`state`, + lopende uur doorgetrokken). +- C++ (clang++ -std=c++17 -Wall -Wextra -Werror -I.): 13/13 PASS (ongewijzigd). +- `generate_packages.py --check` groen; bench-kopieën opnieuw gegenereerd. +- ESPHome 2026.6.2 compile: `easy-guition-device.yaml` SUCCESS (RAM 32,5%, flash + 18,6%), `easy-cyd-device.yaml` SUCCESS (RAM 31,5%, flash 77,4%); alleen de bekende + enum-waarschuwing op `runtime_tiles.h:596`. + +## Op de bordjes (USB, beide op de bench; HA 2026.9.1 op de Yellow) + +- Firmware rechtstreeks via de ESPHome-API (`screen_message`-actie, beide borden): + layout → `Indeling ontvangen`, states → `Tegels laden`/`Gesynchroniseerd`, + dezelfde layout nogmaals → `Gesynchroniseerd` (geen wissel), ping met juiste + revisie → `Gesynchroniseerd`, verkeerde revisie → `Indeling opnieuw nodig`, ping + zonder `rev` → `Fout: ongeldig bericht`, bericht van 3 KB in één actie → verwerkt. +- Add-on 0.2.39 als lokale add-on op de Yellow (via de SMB-share en de Supervisor), + GitHub-add-on 0.2.38 tijdelijk gestopt, de echte indelingen (CYD 8 tegels met drie + sensortegels en weer, Guition 9 tegels) via ingress opgeslagen: beide schermen + `Gesynchroniseerd`, Guition-render (`capture_ui.py`) toont Woonkamer met live + waarden. +- Eerste poging met `Uptime` als `type: timestamp` onder dezelfde naam faalde in HA: + "has a unit of measurement ... non-numeric device class: timestamp" (het register + houdt eenheid `s` vast). Opgelost met de nieuwe entiteit `Opgestart`; na de flash + toont die een tijdstempel en verwijdert HA `sensor._uptime` zelf. +- Herstart (USB-flash van beide borden terwijl de monitor liep): inbox + `Klaar voor tegelconfiguratie` op t=41 s, volledige herzending op t=42 s (10 + berichten CYD, 11 Guition), `Gesynchroniseerd` binnen 1 s; daarna pings op + t=168 s (beide), ertussen alleen de gewijzigde tegel `sensor.3d_printer_vermogen`. + Alle verkeer als `esphome.*_screen_message`, nul `text.set_value`. +- Statistieken: twee van de drie CYD-sensoren hebben statistiek (23 uurrijen), + `sensor.3d_printer_netfrequentie` is `unavailable` (geen rijen) en loopt via de + REST-terugval; geen waarschuwingen in het add-onlog. + +## Lange-termijntest (lokale add-on 0.2.39, beide schermen, HA 2026.9.1) + +- 35 minuten (2100 s) gemonitord via de HA-websocket (`call_service` en + `state_changed`): 100 berichten in totaal, 70 statusberichten (alleen de gewijzigde + tegels, vooral `sensor.3d_printer_vermogen` op de CYD, dat elke paar seconden + verandert, en lampen op de Guition) en 30 pings (15 per scherm); nul + layout-herzendingen, nul wissels van de inbox-status, nul beschikbaarheidswissels; + geen `text.set_value`. Add-oncontainer: 0,0% CPU, 41 MB. Schermdiagnostiek in HA: + alleen heap (per 5 min) en wifi-signaal; de instellingsgetallen bleven stil. +- Gevonden en opgelost: na de registerverversing van de app (elke 10 min) werd alles + herbouwd zonder verschil, wat de pingtimer resette; de ping kwam dan na ~210 s in + plaats van ~120 s. Nu volgt `last` alleen een echte volledige verzending en `pinged` + elk verzonden bericht (elk bericht ververst het feed-venster van het scherm). + Regressietest in `test_scaling.py`. Add-on herbouwd en nog 700 s gemonitord: per + scherm was de langste stilte 122 s (CYD) en 127 s (Guition), ruim binnen het + venster van 300 s van de firmware. +- Guition-capture na de hele test: pagina 2/2 met live waarden (script "Koppel + Speakers" Laatst 20:51, dezelfde minuut als de `media_player.join`-aanroepen in de + monitor). +- Bekend, niet uit deze ronde: `weather.buienradar` ondersteunt geen uurverwachting; + de `weather.get_forecasts`-aanroep daarvoor logt elke 30 minuten een ERROR in de + HA-kernlog (gedrag sinds 0.2.23). `Opgestart` toonde op beide borden een + tijdstempel dat ongeveer een minuut na het moment van ontvangst ligt (ESPHome's + eigen berekening); stabiel, niet aangepast. +- Afsluiting: lokale add-on gestopt, GitHub-add-on 0.2.38 weer gestart (werkt ook + met firmware 0.2.33: tekstblokjes en volledige herhaling). Max werkt de add-on in + de winkel bij naar 0.2.39. diff --git a/guition-4848s040.yaml b/guition-4848s040.yaml index 3b90f5b..15d501e 100644 --- a/guition-4848s040.yaml +++ b/guition-4848s040.yaml @@ -1,7 +1,7 @@ # Guition ESP32-S3-4848S040 / 4-inch EU wallbox, 480x480 IPS + GT911. # Use guition-device.yaml; see docs/GUITION.md. CYD hardware is a separate profile. substitutions: - SCREEN_FIRMWARE_VERSION: "0.2.32" + SCREEN_FIRMWARE_VERSION: "0.2.33" # 0: no drift limit; LVGL decides, releasing inside the tile is a tap, leaving it is not # (like iOS). The CYD keeps a limit because its resistive panel jumps. TOUCH_MOVE_LIMIT_PX: "0" @@ -878,6 +878,13 @@ api: - script.execute: id: alert_dismiss reason: "remote" + # ESP Screen Manager 0.2.39+ hands a whole tile message over in one call instead of + # base64 chunks in the text inbox; the inbox keeps showing the result. + - action: screen_message + variables: + message: string + then: + - lambda: 'id(dashboard_inbox).publish_state(runtime_tiles::receive(message));' encryption: key: !secret api_encryption_key @@ -1359,7 +1366,8 @@ font: # Geheugen zichtbaar maken: dit bord heeft geen PSRAM, dus de vrije heap # bepaalt hoeveel UI erbij kan. Verschijnt ook als entity in Home Assistant. debug: - update_interval: 30s + # Heap figures every five minutes: enough to spot a leak, few recorder rows. + update_interval: 300s sensor: - platform: debug @@ -1378,10 +1386,13 @@ sensor: name: "Wifi Signal" update_interval: 600s + # When the screen booted, as a timestamp: one value per boot instead of a new recorder + # row every 15 s. A new entity on purpose: Home Assistant keeps the old Uptime's unit + # "s" in its registry and would refuse a timestamp under that name. - platform: uptime - name: "Uptime" - id: uptime_s - update_interval: 15s + type: timestamp + name: "Opgestart" + id: boot_time - platform: homeassistant id: tile1_brightness @@ -5226,6 +5237,11 @@ script: if (settings.show_clock) lv_obj_clear_flag(id(lbl_time), LV_OBJ_FLAG_HIDDEN); else lv_obj_add_flag(id(lbl_time), LV_OBJ_FLAG_HIDDEN); } + // The four setting entities only publish a changed value (HA keeps identical states anyway). + if (id(setting_brightness).state != settings.brightness) id(setting_brightness).publish_state(settings.brightness); + if (id(setting_standby_brightness).state != settings.standby_brightness) id(setting_standby_brightness).publish_state(settings.standby_brightness); + if (id(setting_night_brightness).state != settings.night_brightness) id(setting_night_brightness).publish_state(settings.night_brightness); + if (id(setting_standby_seconds).state != settings.standby_seconds) id(setting_standby_seconds).publish_state(settings.standby_seconds); if (!id(ui_refresh).is_running()) id(ui_refresh).execute(); - id: close_cards @@ -6822,7 +6838,7 @@ number: max_value: 100 step: 1 unit_of_measurement: "%" - update_interval: 2s + update_interval: never lambda: return screen_settings::current.brightness; set_action: - lambda: |- @@ -6841,7 +6857,7 @@ number: max_value: 100 step: 1 unit_of_measurement: "%" - update_interval: 2s + update_interval: never lambda: return screen_settings::current.standby_brightness; set_action: - lambda: |- @@ -6860,7 +6876,7 @@ number: max_value: 100 step: 1 unit_of_measurement: "%" - update_interval: 2s + update_interval: never lambda: return screen_settings::current.night_brightness; set_action: - lambda: |- @@ -6879,7 +6895,7 @@ number: max_value: 86400 step: 60 unit_of_measurement: "s" - update_interval: 2s + update_interval: never lambda: return screen_settings::current.standby_seconds; set_action: - lambda: |- diff --git a/home-like-2432s028.yaml b/home-like-2432s028.yaml index 6cfc828..fe13620 100644 --- a/home-like-2432s028.yaml +++ b/home-like-2432s028.yaml @@ -58,7 +58,7 @@ ############################################################ substitutions: - SCREEN_FIRMWARE_VERSION: "0.2.32" + SCREEN_FIRMWARE_VERSION: "0.2.33" # A tap survives a finger that drifts up to this far (320 px over 58 mm: 1 cm); further is a swipe. TOUCH_MOVE_LIMIT_PX: "56" # Shortest contact that counts as a tap; a resistive panel bounces, a capacitive one does not. @@ -1073,6 +1073,13 @@ api: - script.execute: id: alert_dismiss reason: "remote" + # ESP Screen Manager 0.2.39+ hands a whole tile message over in one call instead of + # base64 chunks in the text inbox; the inbox keeps showing the result. + - action: screen_message + variables: + message: string + then: + - lambda: 'id(dashboard_inbox).publish_state(runtime_tiles::receive(message));' encryption: key: !secret api_encryption_key @@ -1549,7 +1556,8 @@ font: # Geheugen zichtbaar maken: dit bord heeft geen PSRAM, dus de vrije heap # bepaalt hoeveel UI erbij kan. Verschijnt ook als entity in Home Assistant. debug: - update_interval: 30s + # Heap figures every five minutes: enough to spot a leak, few recorder rows. + update_interval: 300s sensor: - platform: debug @@ -1568,10 +1576,13 @@ sensor: name: "Wifi Signal" update_interval: 600s + # When the screen booted, as a timestamp: one value per boot instead of a new recorder + # row every 15 s. A new entity on purpose: Home Assistant keeps the old Uptime's unit + # "s" in its registry and would refuse a timestamp under that name. - platform: uptime - name: "Uptime" - id: uptime_s - update_interval: 15s + type: timestamp + name: "Opgestart" + id: boot_time - platform: homeassistant id: tile1_brightness @@ -5326,6 +5337,11 @@ script: if (settings.show_clock) lv_obj_clear_flag(id(lbl_time), LV_OBJ_FLAG_HIDDEN); else lv_obj_add_flag(id(lbl_time), LV_OBJ_FLAG_HIDDEN); } + // The four setting entities only publish a changed value (HA keeps identical states anyway). + if (id(setting_brightness).state != settings.brightness) id(setting_brightness).publish_state(settings.brightness); + if (id(setting_standby_brightness).state != settings.standby_brightness) id(setting_standby_brightness).publish_state(settings.standby_brightness); + if (id(setting_night_brightness).state != settings.night_brightness) id(setting_night_brightness).publish_state(settings.night_brightness); + if (id(setting_standby_seconds).state != settings.standby_seconds) id(setting_standby_seconds).publish_state(settings.standby_seconds); if (!id(ui_refresh).is_running()) id(ui_refresh).execute(); - id: close_cards @@ -6927,7 +6943,7 @@ number: max_value: 100 step: 1 unit_of_measurement: "%" - update_interval: 2s + update_interval: never lambda: return screen_settings::current.brightness; set_action: - lambda: |- @@ -6946,7 +6962,7 @@ number: max_value: 100 step: 1 unit_of_measurement: "%" - update_interval: 2s + update_interval: never lambda: return screen_settings::current.standby_brightness; set_action: - lambda: |- @@ -6965,7 +6981,7 @@ number: max_value: 100 step: 1 unit_of_measurement: "%" - update_interval: 2s + update_interval: never lambda: return screen_settings::current.night_brightness; set_action: - lambda: |- @@ -6984,7 +7000,7 @@ number: max_value: 86400 step: 60 unit_of_measurement: "s" - update_interval: 2s + update_interval: never lambda: return screen_settings::current.standby_seconds; set_action: - lambda: |- diff --git a/packages/cyd.yaml b/packages/cyd.yaml index bce7b87..fc35cb3 100644 --- a/packages/cyd.yaml +++ b/packages/cyd.yaml @@ -59,7 +59,7 @@ ############################################################ substitutions: - SCREEN_FIRMWARE_VERSION: "0.2.32" + SCREEN_FIRMWARE_VERSION: "0.2.33" # A tap survives a finger that drifts up to this far (320 px over 58 mm: 1 cm); further is a swipe. TOUCH_MOVE_LIMIT_PX: "56" # Shortest contact that counts as a tap; a resistive panel bounces, a capacitive one does not. @@ -1077,6 +1077,13 @@ api: - script.execute: id: alert_dismiss reason: "remote" + # ESP Screen Manager 0.2.39+ hands a whole tile message over in one call instead of + # base64 chunks in the text inbox; the inbox keeps showing the result. + - action: screen_message + variables: + message: string + then: + - lambda: 'id(dashboard_inbox).publish_state(runtime_tiles::receive(message));' captive_portal: @@ -1538,7 +1545,8 @@ font: # Geheugen zichtbaar maken: dit bord heeft geen PSRAM, dus de vrije heap # bepaalt hoeveel UI erbij kan. Verschijnt ook als entity in Home Assistant. debug: - update_interval: 30s + # Heap figures every five minutes: enough to spot a leak, few recorder rows. + update_interval: 300s sensor: - platform: debug @@ -1557,10 +1565,13 @@ sensor: name: "Wifi Signal" update_interval: 600s + # When the screen booted, as a timestamp: one value per boot instead of a new recorder + # row every 15 s. A new entity on purpose: Home Assistant keeps the old Uptime's unit + # "s" in its registry and would refuse a timestamp under that name. - platform: uptime - name: "Uptime" - id: uptime_s - update_interval: 15s + type: timestamp + name: "Opgestart" + id: boot_time - platform: template update_interval: never @@ -5257,6 +5268,11 @@ script: if (settings.show_clock) lv_obj_clear_flag(id(lbl_time), LV_OBJ_FLAG_HIDDEN); else lv_obj_add_flag(id(lbl_time), LV_OBJ_FLAG_HIDDEN); } + // The four setting entities only publish a changed value (HA keeps identical states anyway). + if (id(setting_brightness).state != settings.brightness) id(setting_brightness).publish_state(settings.brightness); + if (id(setting_standby_brightness).state != settings.standby_brightness) id(setting_standby_brightness).publish_state(settings.standby_brightness); + if (id(setting_night_brightness).state != settings.night_brightness) id(setting_night_brightness).publish_state(settings.night_brightness); + if (id(setting_standby_seconds).state != settings.standby_seconds) id(setting_standby_seconds).publish_state(settings.standby_seconds); if (!id(ui_refresh).is_running()) id(ui_refresh).execute(); - id: close_cards @@ -6858,7 +6874,7 @@ number: max_value: 100 step: 1 unit_of_measurement: "%" - update_interval: 2s + update_interval: never lambda: return screen_settings::current.brightness; set_action: - lambda: |- @@ -6877,7 +6893,7 @@ number: max_value: 100 step: 1 unit_of_measurement: "%" - update_interval: 2s + update_interval: never lambda: return screen_settings::current.standby_brightness; set_action: - lambda: |- @@ -6896,7 +6912,7 @@ number: max_value: 100 step: 1 unit_of_measurement: "%" - update_interval: 2s + update_interval: never lambda: return screen_settings::current.night_brightness; set_action: - lambda: |- @@ -6915,7 +6931,7 @@ number: max_value: 86400 step: 60 unit_of_measurement: "s" - update_interval: 2s + update_interval: never lambda: return screen_settings::current.standby_seconds; set_action: - lambda: |- diff --git a/packages/guition.yaml b/packages/guition.yaml index 00cf1d4..4a6ce1b 100644 --- a/packages/guition.yaml +++ b/packages/guition.yaml @@ -2,7 +2,7 @@ # Guition ESP32-S3-4848S040 / 4-inch EU wallbox, 480x480 IPS + GT911. # Use guition-device.yaml; see docs/GUITION.md. CYD hardware is a separate profile. substitutions: - SCREEN_FIRMWARE_VERSION: "0.2.32" + SCREEN_FIRMWARE_VERSION: "0.2.33" # 0: no drift limit; LVGL decides, releasing inside the tile is a tap, leaving it is not # (like iOS). The CYD keeps a limit because its resistive panel jumps. TOUCH_MOVE_LIMIT_PX: "0" @@ -882,6 +882,13 @@ api: - script.execute: id: alert_dismiss reason: "remote" + # ESP Screen Manager 0.2.39+ hands a whole tile message over in one call instead of + # base64 chunks in the text inbox; the inbox keeps showing the result. + - action: screen_message + variables: + message: string + then: + - lambda: 'id(dashboard_inbox).publish_state(runtime_tiles::receive(message));' captive_portal: @@ -1348,7 +1355,8 @@ font: # Geheugen zichtbaar maken: dit bord heeft geen PSRAM, dus de vrije heap # bepaalt hoeveel UI erbij kan. Verschijnt ook als entity in Home Assistant. debug: - update_interval: 30s + # Heap figures every five minutes: enough to spot a leak, few recorder rows. + update_interval: 300s sensor: - platform: debug @@ -1367,10 +1375,13 @@ sensor: name: "Wifi Signal" update_interval: 600s + # When the screen booted, as a timestamp: one value per boot instead of a new recorder + # row every 15 s. A new entity on purpose: Home Assistant keeps the old Uptime's unit + # "s" in its registry and would refuse a timestamp under that name. - platform: uptime - name: "Uptime" - id: uptime_s - update_interval: 15s + type: timestamp + name: "Opgestart" + id: boot_time - platform: template update_interval: never @@ -5157,6 +5168,11 @@ script: if (settings.show_clock) lv_obj_clear_flag(id(lbl_time), LV_OBJ_FLAG_HIDDEN); else lv_obj_add_flag(id(lbl_time), LV_OBJ_FLAG_HIDDEN); } + // The four setting entities only publish a changed value (HA keeps identical states anyway). + if (id(setting_brightness).state != settings.brightness) id(setting_brightness).publish_state(settings.brightness); + if (id(setting_standby_brightness).state != settings.standby_brightness) id(setting_standby_brightness).publish_state(settings.standby_brightness); + if (id(setting_night_brightness).state != settings.night_brightness) id(setting_night_brightness).publish_state(settings.night_brightness); + if (id(setting_standby_seconds).state != settings.standby_seconds) id(setting_standby_seconds).publish_state(settings.standby_seconds); if (!id(ui_refresh).is_running()) id(ui_refresh).execute(); - id: close_cards @@ -6753,7 +6769,7 @@ number: max_value: 100 step: 1 unit_of_measurement: "%" - update_interval: 2s + update_interval: never lambda: return screen_settings::current.brightness; set_action: - lambda: |- @@ -6772,7 +6788,7 @@ number: max_value: 100 step: 1 unit_of_measurement: "%" - update_interval: 2s + update_interval: never lambda: return screen_settings::current.standby_brightness; set_action: - lambda: |- @@ -6791,7 +6807,7 @@ number: max_value: 100 step: 1 unit_of_measurement: "%" - update_interval: 2s + update_interval: never lambda: return screen_settings::current.night_brightness; set_action: - lambda: |- @@ -6810,7 +6826,7 @@ number: max_value: 86400 step: 60 unit_of_measurement: "s" - update_interval: 2s + update_interval: never lambda: return screen_settings::current.standby_seconds; set_action: - lambda: |- diff --git a/screen_manager/CHANGELOG.md b/screen_manager/CHANGELOG.md index 700ca25..3c23156 100644 --- a/screen_manager/CHANGELOG.md +++ b/screen_manager/CHANGELOG.md @@ -1,3 +1,14 @@ +## 0.2.39 (firmware 0.2.33) + +Zuiniger in dagelijks gebruik, ook met tientallen schermen. Niets verandert aan wat je ziet of instelt. + +- **Alleen doorrekenen wat veranderde.** Bij een statuswijziging bouwt de app alleen de tegels (en de bovenbalk) opnieuw waar die entiteit op staat; de andere berichten blijven zoals ze verstuurd zijn. De schermlijst wordt niet meer bij elke wijziging uit het hele Home Assistant-register opgebouwd, maar bewaard tot het register of een schermdiagnose verandert. De live-stream van de beheerpagina volgt dezelfde regel. +- **Historie gebundeld, buiten de lus.** De 24 staafjes van sensortegels komen uit één `recorder/statistics_during_period`-vraag voor alle sensoren tegelijk (uurgemiddelden voor 24 uur, 5-minuutgemiddelden voor 1 en 6 uur), in een achtergrondtaak die elke vijf minuten ververst. Sensoren zonder statistiek (geen state class) houden de oude REST-historie, ook op de achtergrond. De verzendlus wacht nergens meer op de database; een gewijzigde grafiek stuurt alleen die tegel. +- **Keepalive als ping.** Schermen met firmware 0.2.33 krijgen elke twee minuten alleen een klein bericht met de revisie van hun indeling. Komt die niet overeen (herstart, demo-indeling, verloren bericht), dan meldt het scherm `Indeling opnieuw nodig` en stuurt de app alles opnieuw; als vangnet gaat eens per uur toch alles. De inbox-status wisselt niet meer per ronde tussen "Indeling ontvangen" en "Gesynchroniseerd". Oudere firmware houdt de volledige herhaling per twee minuten. +- **Eén actie per bericht.** Firmware 0.2.33 heeft de actie `esphome._screen_message` die een heel bericht in één keer aanneemt; de app gebruikt die zodra het scherm die firmware meldt. De base64-blokjes van 200 tekens in de tekstentiteit blijven de terugval voor oudere firmware. +- **Rustige diagnostiek.** De diagnostische sensor `Uptime` (elke 15 s een nieuwe waarde) is vervangen door `Opgestart`, een tijdstempel met één waarde per opstart; Home Assistant ruimt de oude entiteit zelf op. De heap-sensoren melden per vijf minuten in plaats van per 30 s, en de vier instellingsgetallen (helderheid, standby) publiceren alleen nog een gewijzigde waarde. Dat scheelt tot honderdduizenden recorder-rijen per dag bij dertig schermen. +- Firmware 0.2.33 voor beide borden; de app biedt de update aan. Alles werkt ook met de vorige firmware, alleen zonder ping en actie. + ## 0.2.38 (firmware 0.2.32) - **Bovenbalk per scherm.** Bovenaan elk scherm staat links de naam en rechts wat jij kiest, tot zes onderdelen: de **tijd**, een **analoge klok**, de **datum**, of een **entiteit met icoon** uit Home Assistant, zoals temperatuur, luchtvochtigheid, verbruik, een deur of raam (open/dicht), het alarmpaneel, een slot, wie er thuis is (`zone.home`) of een telefoon (`device_tracker`). Zonder aanpassing blijft het zoals het was: naam en tijd. diff --git a/screen_manager/app/core.py b/screen_manager/app/core.py index a77ea03..993e7f1 100644 --- a/screen_manager/app/core.py +++ b/screen_manager/app/core.py @@ -1,6 +1,7 @@ """Pure validation, firmware generation and bounded display protocol.""" import base64 from datetime import datetime, timedelta, timezone +import hashlib import json import math import re @@ -16,7 +17,7 @@ WEEKDAYS = ['ma', 'di', 'wo', 'do', 'vr', 'za', 'zo'] REPO = 'https://github.com/MaxGramser/homeassistant_espscreen' REFS = {'cyd': 'main', 'guition': 'main'} # Firmware shipped with this app release; screens below it get an update offer. -FIRMWARE_VERSION = '0.2.32' +FIRMWARE_VERSION = '0.2.33' ATTRS = frozenset('brightness percentage current_position current_temperature temperature current_humidity min_temp max_temp target_temp_step supported_color_modes hvac_modes hvac_action hs_color color_temp_kelvin min_color_temp_kelvin max_color_temp_kelvin fan_speed_list unit_of_measurement battery_level fan_speed volume_level is_volume_muted media_title options min max step temperature_unit supported_features device_class next_rising next_setting finishes_at duration remaining humidity wind_speed wind_speed_unit apparent_temperature fan_modes swing_modes fan_mode swing_mode'.split()) # Attributes whose boolean value the screen needs; every other bool stays behind. BOOL_ATTRS = frozenset(['is_volume_muted']) @@ -109,6 +110,9 @@ def resolve_controls(tile): choice = options.get('controls', CONTROLS[domain][0][0]) return None if choice == 'none' else choice +# Diagnostic entities every ESP Screens firmware exposes; the manager watches them for screens. +SCREEN_ENTITY_NAMES = frozenset({'Tegelinstellingen', 'Schermfirmware', 'Guition schermtype', 'Apparaatnaam', 'IP-adres'}) + # Additive schema 1 extension. An absent object retains old firmware/YAML defaults. SETTING_RULES = { 'standby_enabled': (True, None, None), @@ -152,6 +156,11 @@ def entity_id(value): # ----- Top bar (firmware 0.2.32+): the screen name on the left, up to six items on the right. # Without a `header` the screen keeps its name and the clock of `show_clock`. ----- HEADER_MIN_FIRMWARE = (0, 2, 32) +# Firmware 0.2.33+ takes a whole message in one API action (esphome._screen_message) +# and answers a keepalive ping with its layout revision; older firmware gets base64 +# chunks in the text inbox and a full repeat every keepalive. +TRANSPORT_MIN_FIRMWARE = (0, 2, 33) +MESSAGE_ACTION = 'screen_message' HEADER_MAX_ITEMS = 6 # Items the screen draws on its own clock, without Home Assistant. HEADER_BUILTIN = {'clock': 'Tijd', 'analog': 'Analoge klok', 'date': 'Datum'} @@ -438,15 +447,28 @@ def state_message(index, tile, states, extra=None): 'state': short(state.get('state', 'unavailable'), 160), 'a': bounded, **({'o': options} if options is not None else {}), **({'x': extra} if extra else {})} -def packets(message, token=None): - raw = json.dumps(message, ensure_ascii=False, separators=(',', ':'), allow_nan=False).encode() - if len(raw) > 4096: +def encode(message): + """The message as the firmware parses it: compact JSON, at most 4096 bytes.""" + raw = json.dumps(message, ensure_ascii=False, separators=(',', ':'), allow_nan=False) + if len(raw.encode()) > 4096: raise ValueError('Het schermbericht is te groot.') - encoded = base64.b64encode(raw).decode('ascii') + return raw + +def revision(message): + """Short fingerprint of a layout message; the screen echoes it back on every ping.""" + return hashlib.sha256(json.dumps(message, sort_keys=True, separators=(',', ':')).encode()).hexdigest()[:12] + +def packets(message, token=None): + """The message as base64 chunks for the 255-character text inbox (firmware before 0.2.33).""" + encoded = base64.b64encode(encode(message).encode()).decode('ascii') token = token or secrets.token_hex(8) chunks = [encoded[i:i+200] for i in range(0, len(encoded), 200)] return [f'{token}|{i}|{int(i == len(chunks)-1)}|{part}' for i, part in enumerate(chunks)] +def message_action(node): + """Home Assistant action that hands a screen one whole message, or None without a node name.""" + return alert_service(node, MESSAGE_ACTION) + # ----- Alerts (firmware 0.2.31+): the reference the cheatsheet shows. tests/test_alerts_reference.py # keeps every value here equal to what the board profiles compile. ----- ALERT_MIN_FIRMWARE = '0.2.31' @@ -485,7 +507,12 @@ def alert_reference(): 'suggested_icons': [{'name': name, 'cp': tile_icons.GLYPHS[name]} for name in ALERT_SUGGESTED_ICONS], 'extra_icons': [{'name': name, 'cp': cp} for name, cp in tile_icons.FIXED]} -def discover(registry, states, devices, areas): +def screen_items(registry): + """The ESPHome entities that describe a screen; the subset `discover_screens` needs.""" + return [item for item in registry if item.get('platform') == 'esphome' and item.get('original_name') in SCREEN_ENTITY_NAMES] + +def discover_screens(registry, states, devices, areas): + """Paired screens: every enabled ESPHome inbox with the diagnostics of its device.""" device_map = {d['id']: d for d in devices} area_map = {a['area_id']: a['name'] for a in areas} versions = {item.get('device_id'): states.get(item['entity_id'], {}).get('state', 'onbekend') for item in registry @@ -502,23 +529,37 @@ def discover(registry, states, devices, areas): return found nodes = diagnostic('Apparaatnaam', r'[a-z0-9][a-z0-9-]{0,30}') addresses = diagnostic('IP-adres', r'\d{1,3}(\.\d{1,3}){3}') - screens, entities = [], [] + screens = [] for item in registry: eid = item['entity_id'] + if not (item.get('platform') == 'esphome' and eid.startswith('text.') and item.get('original_name') == 'Tegelinstellingen' and not item.get('disabled_by')): + continue device = device_map.get(item.get('device_id'), {}) state = states.get(eid, {}) area = area_map.get(item.get('area_id') or device.get('area_id'), '') - if item.get('platform') == 'esphome' and eid.startswith('text.') and item.get('original_name') == 'Tegelinstellingen' and not item.get('disabled_by'): - screens.append({'id': eid, 'name': device.get('name_by_user') or device.get('name') or eid, - 'firmware': versions.get(item.get('device_id'), 'onbekend'), - 'board': boards.get(item.get('device_id'), 'unknown'), - 'node': nodes.get(item.get('device_id')), 'ip': addresses.get(item.get('device_id')), - 'device': device.get('name') or '', - 'area': area, 'online': state.get('state') not in (None, 'unknown', 'unavailable'), - 'status': state.get('state', 'Niet verbonden')}) + screens.append({'id': eid, 'name': device.get('name_by_user') or device.get('name') or eid, + 'firmware': versions.get(item.get('device_id'), 'onbekend'), + 'board': boards.get(item.get('device_id'), 'unknown'), + 'node': nodes.get(item.get('device_id')), 'ip': addresses.get(item.get('device_id')), + 'device': device.get('name') or '', + 'area': area, 'online': state.get('state') not in (None, 'unknown', 'unavailable'), + 'status': state.get('state', 'Niet verbonden')}) + return screens + +def discover(registry, states, devices, areas): + """(screens, entities): the paired screens and every entity a tile or the top bar can show.""" + device_map = {d['id']: d for d in devices} + area_map = {a['area_id']: a['name'] for a in areas} + screens = discover_screens(registry, states, devices, areas) + entities = [] + for item in registry: + eid = item['entity_id'] tile = entity_id(eid) if not (tile or header_entity(eid)) or item.get('disabled_by'): continue + device = device_map.get(item.get('device_id'), {}) + state = states.get(eid, {}) + area = area_map.get(item.get('area_id') or device.get('area_id'), '') entities.append({'id': eid, 'name': state.get('attributes', {}).get('friendly_name') or item.get('name') or item.get('original_name') or eid, 'device': device.get('name_by_user') or device.get('name') or '', 'area': area, 'state': state.get('state', 'unavailable'), 'icon': tile_icons.ha_icon(state.get('attributes')), diff --git a/screen_manager/app/server.py b/screen_manager/app/server.py index 18df3f2..3185cbf 100644 --- a/screen_manager/app/server.py +++ b/screen_manager/app/server.py @@ -1,4 +1,4 @@ -"""HA Ingress app. HA writes: text.set_value on discovered inboxes and one persistent notification when a nightly update stops.""" +"""HA Ingress app. HA writes: text.set_value on discovered inboxes (or the screen_message action on firmware 0.2.33+) and one persistent notification when a nightly update stops.""" import asyncio import contextlib import json @@ -14,17 +14,45 @@ import tile_icons from updates import Updater from aiohttp import ClientSession, ClientTimeout, WSMsgType, web -from core import BUILTIN, HEADER_MIN_FIRMWARE, TILE_BACKGROUNDS, alert_reference, alert_service, controls_catalogue, discover, extras, header_items, min_firmware, pack_slots, packets, state_message, validate_header, validate_layout, validate_settings +from core import BUILTIN, HEADER_MIN_FIRMWARE, TILE_BACKGROUNDS, TRANSPORT_MIN_FIRMWARE, alert_reference, alert_service, controls_catalogue, discover, discover_screens, encode, extras, header_items, message_action, min_firmware, pack_slots, packets, revision, screen_items, state_message, validate_header, validate_layout, validate_settings import header_bar from zoneinfo import ZoneInfo LOG = logging.getLogger('screen_manager') REGISTRY_EVENTS = ('entity_registry_updated', 'device_registry_updated', 'area_registry_updated') -SCREEN_ENTITY_NAMES = {'Tegelinstellingen', 'Schermfirmware', 'Guition schermtype', 'Apparaatnaam', 'IP-adres'} -# Full repeat of a screen's layout and states. Every layout message declares it, so the -# firmware sizes its feed watchdog from this one constant (firmware 0.2.22+). +# Keepalive cadence. Firmware 0.2.33+ gets a small ping carrying the layout revision; every +# layout message declares this number, so the firmware sizes its feed watchdog from it +# (firmware 0.2.22+). Firmware before 0.2.33 still gets the whole layout every round. KEEPALIVE_SECONDS = 120 +# Safety net for screens that answer every ping: everything again once an hour. +FULL_REPEAT_SECONDS = 3600 +# Sensor history is refreshed in a background task, never inside the sync loop. +HISTORY_SECONDS = 300 +# Inbox states after which a screen needs the whole layout again: a restart, a ping that +# did not match, or a tile state that never arrived. Guarded so a slow batch cannot loop. +RESEND_STATES = frozenset({'Klaar voor tegelconfiguratie', 'Indeling opnieuw nodig', 'Tegels laden'}) +RESEND_GUARD_SECONDS = 120 +FORECAST_SECONDS = 1800 + +def samples(events, begin, span): + """24 values, one per bucket: the last known value at the end of each bucket (None until the first).""" + events.sort(key=lambda pair: pair[0]) + position, value, out = 0, None, [] + for bucket in range(24): + boundary = begin + (bucket + 1) * span / 24 + while position < len(events) and events[position][0] <= boundary: + value = events[position][1] + position += 1 + out.append(value) + return out + +def rounded(value): + try: + value = float(value) + except (TypeError, ValueError): + return None + return round(value, 3) if math.isfinite(value) else None class HomeAssistant: registry_interval = 600 @@ -40,6 +68,8 @@ class HomeAssistant: self.changed = asyncio.Event() # Entity ids the manager cares about; None wakes it for every state change. self.relevant = None + # Entity ids whose state changed since the manager last looked; it only rebuilds those tiles. + self.dirty = set() self.registry_changed = asyncio.Event() self.setting_events = [] self.time_zone = None @@ -84,6 +114,7 @@ class HomeAssistant: else: self.states.pop(eid, None) if self.relevant is None or eid in self.relevant: + self.dirty.add(eid) self.changed.set() elif event.get('event_type') in REGISTRY_EVENTS: self.registry_changed.set() @@ -168,7 +199,12 @@ class HomeAssistant: future.set_exception(ConnectionError('Verbinding verbroken.')) await asyncio.sleep(5) - async def send(self, inbox, message): + async def send(self, inbox, message, action=None): + """One message to a screen: as one ESPHome action call (firmware 0.2.33+), else as text chunks.""" + if action: + domain, service = action.split('.', 1) + await self.request('call_service', domain=domain, service=service, service_data={'message': encode(message)}) + return for packet in packets(message): await self.request('call_service', domain='text', service='set_value', service_data={'entity_id': inbox, 'value': packet}) @@ -180,6 +216,30 @@ class HomeAssistant: forecast = ((result or {}).get('response') or {}).get(entity, {}).get('forecast', []) return forecast if isinstance(forecast, list) else [] + async def statistics(self, entities, hours): + """24 samples per entity from the recorder's statistics, all entities in one request. + + Hourly means for a day, five-minute means below that; a sum-only sensor (energy) gives its + state. Entities without statistics (no state class) are absent and fall back to `history`.""" + period = 'hour' if hours >= 24 else '5minute' + start = datetime.now(timezone.utc) - timedelta(hours=hours) + result = await self.request('recorder/statistics_during_period', start_time=start.isoformat(), + statistic_ids=sorted(entities), period=period, types=['mean', 'state']) + begin, span, found = start.timestamp(), hours * 3600, {} + for entity, rows in (result or {}).items(): + if entity not in entities or not isinstance(rows, list): + continue + events = [] + for row in rows: + moment = row.get('start') if isinstance(row, dict) else None + if not isinstance(moment, (int, float)): + continue + if moment > 1e11: # milliseconds since HA 2023.9 + moment /= 1000 + events.append((moment, rounded(row.get('mean') if row.get('mean') is not None else row.get('state')))) + found[entity] = samples(events, begin, span) + return found + async def history(self, entity, hours): start = (datetime.now(timezone.utc)-timedelta(hours=hours)).isoformat() async with self.session.get(self.base+'/history/period/'+start, @@ -191,34 +251,33 @@ class HomeAssistant: raw.extend(chunk) if len(raw)>2*1024*1024: raise ValueError('Geschiedenis te groot.') rows=json.loads(raw) - samples=[]; events=[] + events=[] begin=datetime.fromisoformat(start).timestamp(); span=hours*3600 for row in rows[0] if rows else []: try: timestamp=datetime.fromisoformat(row.get('last_changed',row.get('last_updated','')).replace('Z','+00:00')).timestamp() except (ValueError,TypeError): continue - try: - value=float(row['state']); value=round(value,3) if math.isfinite(value) else None - except (ValueError,TypeError,KeyError): value=None - events.append((timestamp,value)) - events.sort(key=lambda pair:pair[0]);position=0;value=None - for bucket in range(24): - boundary=begin+(bucket+1)*span/24 - while position (monotonic, 24 samples); filled by history_loop, read by sync_one. self.histories = {} + self.history_wake = asyncio.Event() self.forecasts = {} self.listeners = set() # asyncio.Event per open /api/events stream self.firmware = Firmware(os.environ.get("ESPHOME_CONFIG", "/homeassistant/esphome"), self.path.parent) self.updates = Updater(self, self.path.parent / 'updates.json') self._registry_source, self._registry_index = None, {} + self._items_source, self._items = None, [] + self._screens_key, self._screens = None, [] + self._watched_key, self._watched = None, set() + self._seen_registry = None if self.path.exists(): raw = json.loads(self.path.read_text()) # Versioned persistent data. Never silently overwrite an unknown schema. @@ -227,11 +286,34 @@ class Manager: self.layouts = {key: validate_layout(value) for key, value in raw['screens'].items()} def inventory(self): + """(screens, every tile/top-bar entity): the full walk over the registry, for the editor and saving.""" return discover(self.ha.registry, self.ha.states, self.ha.devices, self.ha.areas) + def screen_registry(self): + """The few registry entries that describe screens; rebuilt only when HA delivers a new registry.""" + registry = getattr(self.ha, 'registry', []) + if self._items_source is not registry: + self._items_source, self._items = registry, screen_items(registry) + return self._items + + def screens(self): + """The paired screens, cached until the registry or one of the screens' own diagnostics changes. + + Costs a handful of dictionary lookups per screen instead of a walk over every entity in + Home Assistant, so the sync loop can run it on every wake.""" + ha = self.ha + items = self.screen_registry() + key = (id(ha.registry), id(ha.devices), id(ha.areas), tuple(ha.states.get(item['entity_id'], {}).get('state') for item in items)) + if key != self._screens_key: + self._screens_key, self._screens = key, discover_screens(items, ha.states, ha.devices, ha.areas) + return [dict(screen) for screen in self._screens] + + def screen(self, inbox): + return next((s for s in self.screens() if s['id'] == inbox), None) + def firmware_version(self, inbox, screen=None): if screen is None: - screen=next((s for s in self.inventory()[0] if s['id']==inbox), {}) + screen = self.screen(inbox) or {} try: version=tuple(int(part) for part in screen.get('firmware','').split('.')) return version if len(version)==3 else None @@ -245,6 +327,16 @@ class Manager: def supports_header(self, inbox, screen=None): return (self.firmware_version(inbox, screen) or (0, 0, 0)) >= HEADER_MIN_FIRMWARE + def supports_ping(self, inbox, screen=None): + """Firmware 0.2.33+ answers a keepalive ping with its layout revision.""" + return (self.firmware_version(inbox, screen) or (0, 0, 0)) >= TRANSPORT_MIN_FIRMWARE + + def transport(self, inbox, screen=None): + """The ESPHome action that takes a whole message (firmware 0.2.33+), or None for the text inbox.""" + if screen is None: + screen = self.screen(inbox) or {} + return message_action(screen.get('node')) if self.supports_ping(inbox, screen) else None + def registry_index(self): """Entity registry by id (display precision, entity category); rebuilt only when HA delivers a new registry.""" registry = getattr(self.ha, 'registry', []) @@ -316,6 +408,7 @@ class Manager: self.layouts = updated self.sent.pop(inbox, None) self.status[inbox] = 'Opgeslagen; wacht op synchronisatie' + self.history_wake.set() self.ha.changed.set() self.notify() @@ -337,13 +430,16 @@ class Manager: if meta.get('screen') and meta.get('node') not in nodes and (meta.get('friendly') or None) not in devices] def watched_entities(self): - """Entities whose state changes matter: tiles on any layout plus the screens' own diagnostics.""" - watched = {tile['entity'] for layout in self.layouts.values() for tile in layout['tiles']} - watched |= {item['entity'] for layout in self.layouts.values() for item in header_items(layout) if item['type'] == 'entity'} - for item in getattr(self.ha, 'registry', []): - if item.get('platform') == 'esphome' and item.get('original_name') in SCREEN_ENTITY_NAMES: - watched.add(item['entity_id']) - return watched + """Entities whose state changes matter: tiles on any layout plus the screens' own diagnostics. + + Cached per (registry, layouts): both are replaced as whole objects when they change.""" + key = (id(getattr(self.ha, 'registry', [])), id(self.layouts)) + if key != self._watched_key: + watched = {tile['entity'] for layout in self.layouts.values() for tile in layout['tiles']} + watched |= {item['entity'] for layout in self.layouts.values() for item in header_items(layout) if item['type'] == 'entity'} + watched |= {item['entity_id'] for item in self.screen_registry()} + self._watched_key, self._watched = key, watched + return set(self._watched) async def cached(self, store, key, ttl, fetch): entry=store.get(key) @@ -353,51 +449,120 @@ class Manager: entry=(time.monotonic(),value);store[key]=entry return entry[1] - async def sync_one(self, inbox, layout, force=False, screen=None): + def forecast_due(self, entity): + entry = self.forecasts.get(entity) + hourly = self.forecasts.get((entity, 'hourly')) + return not entry or not hourly or time.monotonic() - min(entry[0], hourly[0]) > FORECAST_SECONDS + + async def tile_message(self, index, tile): + """The state message of one tile: state, options, extras, and the history the background task holds.""" + forecast=hourly=None + if tile['entity'].startswith('weather.') and hasattr(self.ha,'forecast'): + forecast=await self.cached(self.forecasts, tile['entity'], FORECAST_SECONDS, lambda: self.ha.forecast(tile['entity'])) + # Hourly forecasts feed the weather card's next-hours strip (0.2.23+); refreshed every half hour. + hourly=await self.cached(self.forecasts, (tile['entity'],'hourly'), FORECAST_SECONDS, lambda: self.ha.forecast(tile['entity'],'hourly')) + message=state_message(index,tile,self.ha.states,extras(tile,self.ha.states,forecast,getattr(self.ha,'time_zone',None),hourly)) + if tile['entity'].startswith('sensor.') and hasattr(self.ha,'history'): + hours=tile.get('options',{}).get('history_hours',24) + entry=self.histories.get((tile['entity'],hours)) + if entry is not None: + message['history']={'hours':hours,'values':entry[1]} + else: + self.history_wake.set() + return message + + def layout_message(self, inbox, layout, screen): + tiles = layout['tiles'] + # Grid positions (firmware 0.2.26+; older firmware ignores them and packs the entities in order). + message = {'v': 1, 'op': 'layout', 'inbox': inbox, 'title': layout['title'], 'entities': [t['entity'] for t in tiles], + 'slots': [t['slot'] for t in tiles], 'keepalive': KEEPALIVE_SECONDS} + if 'pages' in layout: + message['pages'] = layout['pages'] + if 'settings' in layout: + message['settings'] = {k:v for k,v in layout['settings'].items() if k not in ('swipe_pages','rotation')} + message['swipe_pages'] = layout['settings'].get('swipe_pages',False) + if screen.get('board')=='guition': + message['rotation'] = layout['settings'].get('rotation',0) + # The revision the screen echoes on every ping (firmware 0.2.33+; older firmware ignores it). + message['rev'] = revision(message) + return message + + async def sync_one(self, inbox, layout, force=False, screen=None, dirty=None): + """Send the screen what it lacks; True when something went out. + + `dirty` is the set of entity ids whose state changed since the last pass: only their tiles + (and the top bar, when it shows one of them) are rebuilt, the other messages are reused as + sent. None rebuilds every message and sends the differences; `force` sends everything.""" if screen is None: - screen=next((s for s in self.inventory()[0] if s['id']==inbox), {}) + screen = self.screen(inbox) or {} needed = self.needs_firmware(inbox, layout, screen) if needed: self.status[inbox]=f"Indeling bewaard; firmware {needed}+ nodig voor deze tegels" - return + return False tiles = layout['tiles'] - # Grid positions (firmware 0.2.26+; older firmware ignores them and packs the entities in order). - messages = [{'v': 1, 'op': 'layout', 'inbox': inbox, 'title': layout['title'], 'entities': [t['entity'] for t in tiles], - 'slots': [t['slot'] for t in tiles], 'keepalive': KEEPALIVE_SECONDS}] - if 'pages' in layout: - messages[0]['pages'] = layout['pages'] - if 'settings' in layout: - messages[0]['settings'] = {k:v for k,v in layout['settings'].items() if k not in ('swipe_pages','rotation')} - messages[0]['swipe_pages'] = layout['settings'].get('swipe_pages',False) - if screen.get('board')=='guition': - messages[0]['rotation'] = layout['settings'].get('rotation',0) + layout_msg = self.layout_message(inbox, layout, screen) + previous = self.sent.get(inbox) + full = force or not previous or previous['layout'] != layout_msg or dirty is None # The top bar right after the layout (firmware 0.2.32+; older firmware draws the clock of show_clock). + header_msg = None if self.supports_header(inbox, screen): - messages.append(self.header_message(layout)) - for i,tile in enumerate(layout['tiles']): - forecast=hourly=None - if tile['entity'].startswith('weather.') and hasattr(self.ha,'forecast'): - forecast=await self.cached(self.forecasts, tile['entity'], 1800, lambda: self.ha.forecast(tile['entity'])) - # Hourly forecasts feed the weather card's next-hours strip (0.2.23+); refreshed every half hour. - hourly=await self.cached(self.forecasts, (tile['entity'],'hourly'), 1800, lambda: self.ha.forecast(tile['entity'],'hourly')) - message=state_message(i,tile,self.ha.states,extras(tile,self.ha.states,forecast,getattr(self.ha,'time_zone',None),hourly)) - if tile['entity'].startswith('sensor.') and hasattr(self.ha,'history'): - hours=tile.get('options',{}).get('history_hours',24) - key=(tile['entity'],hours) - message['history']={'hours':hours,'values':await self.cached(self.histories, key, 300, lambda: self.ha.history(*key))} - messages.append(message) - previous = self.sent.get(inbox, []) - force = force or not previous or messages[0] != previous[0] - for i, message in enumerate(messages): + bar = {item['entity'] for item in header_items(layout) if item['type'] == 'entity'} + if full or previous['header'] is None or bar & dirty: + header_msg = self.header_message(layout) + else: + header_msg = previous['header'] + states = [] + for i, tile in enumerate(tiles): + reuse = not full and i < len(previous['states']) and tile['entity'] not in dirty + if reuse and tile['entity'].startswith('weather.') and self.forecast_due(tile['entity']): + reuse = False + states.append(previous['states'][i] if reuse else await self.tile_message(i, tile)) + outgoing = [] + if force or not previous or layout_msg != previous['layout']: + outgoing.append(layout_msg) + if header_msg is not None and (force or not previous or header_msg != previous['header']): + outgoing.append(header_msg) + for i, message in enumerate(states): + if force or not previous or i >= len(previous['states']) or message != previous['states'][i]: + outgoing.append(message) + action = self.transport(inbox, screen) + for message in outgoing: # Save during transmission aborts the old batch, then starts a full new layout. if self.layouts.get(inbox) != layout: self.sent.pop(inbox, None) - return - if force or i >= len(previous) or message != previous[i]: - await self.ha.send(inbox, message) - self.sent[inbox] = messages - self.last[inbox] = time.monotonic() + return True + await self.ha.send(inbox, message, action) + self.sent[inbox] = {'layout': layout_msg, 'header': header_msg, 'states': states, 'rev': layout_msg['rev']} + # The hourly timer follows a real full transmission; any message refreshes the screen's + # feed window, so the ping timer follows whatever went out (a rebuild without differences, + # after a registry refresh, must not postpone the ping). + if force or not previous: + self.last[inbox] = time.monotonic() + if outgoing: + self.pinged[inbox] = time.monotonic() self.status[inbox] = 'Verzonden naar Home Assistant' + return bool(outgoing) + + async def ping(self, inbox, screen): + """Keepalive for firmware 0.2.33+: the layout revision only; the screen asks for the rest itself.""" + sent = self.sent.get(inbox) + if not sent: + return + await self.ha.send(inbox, {'v': 1, 'op': 'ping', 'rev': sent['rev'], 'keepalive': KEEPALIVE_SECONDS}, self.transport(inbox, screen)) + self.pinged[inbox] = time.monotonic() + + def take_dirty(self): + """Entity ids that changed since the last pass, or None when everything must be rebuilt.""" + dirty = getattr(self.ha, 'dirty', None) + if dirty is None: + return None + self.ha.dirty = set() + registry = getattr(self.ha, 'registry', None) + if registry is not self._seen_registry: + # A new registry can change names and display precision: rebuild everything once. + self._seen_registry = registry + return None + return dirty async def run(self): while True: @@ -424,29 +589,95 @@ class Manager: layout['settings']=validate_settings(settings);self.save(inbox,layout) except (ValueError,TypeError): pass if hasattr(self.ha,'setting_events'): self.ha.setting_events.clear() - screens = self.inventory()[0] + dirty = self.take_dirty() + screens_key = self._screens_key + screens = self.screens() + changed = self._screens_key != screens_key self.ha.relevant = self.watched_entities() for screen in screens: inbox = screen['id'] + before = self.status.get(inbox) if not screen['online']: self.sent.pop(inbox, None) self.status[inbox] = 'Scherm offline; wijzigingen bewaard' + changed = changed or self.status[inbox] != before continue if inbox not in self.layouts: continue try: - force = time.monotonic() - self.last.get(inbox, 0) >= KEEPALIVE_SECONDS - # last records full keepalive, not unrelated HA state events. - before = self.last.get(inbox, 0) - await self.sync_one(inbox, self.layouts[inbox], force, screen) - if not force: - self.last[inbox] = before + now = time.monotonic() + pings = self.supports_ping(inbox, screen) + # The screen says it lost the layout (restart, mismatched ping, a state that never came). + if inbox in self.sent and screen['status'] in RESEND_STATES and now - self.last.get(inbox, 0) >= RESEND_GUARD_SECONDS: + self.sent.pop(inbox) + force = now - self.last.get(inbox, 0) >= (FULL_REPEAT_SECONDS if pings else KEEPALIVE_SECONDS) + if await self.sync_one(inbox, self.layouts[inbox], force, screen, dirty): + changed = True + if pings and inbox in self.sent and now - self.pinged.get(inbox, 0) >= KEEPALIVE_SECONDS: + await self.ping(inbox, screen) except Exception as error: self.sent.pop(inbox, None) self.status[inbox] = 'Verzenden mislukt; automatisch opnieuw proberen' LOG.warning('Schermsynchronisatie opnieuw proberen (%s)', type(error).__name__) await asyncio.sleep(1) - self.notify() + changed = changed or self.status.get(inbox) != before + if changed: + self.notify() + + def history_keys(self): + """(entity, hours) of every sensor tile on any screen.""" + return {(tile['entity'], tile.get('options', {}).get('history_hours', 24)) + for layout in self.layouts.values() for tile in layout['tiles'] if tile['entity'].startswith('sensor.')} + + async def refresh_histories(self): + """Fetch the histories that are missing or older than HISTORY_SECONDS: one statistics request per + window for all sensors at once, the REST history only for sensors without statistics. Changed + values mark their entity dirty so the sync loop resends just those tiles.""" + wanted = self.history_keys() + for key in [key for key in self.histories if key not in wanted]: + del self.histories[key] + now = time.monotonic() + due = [key for key in wanted if key not in self.histories or now - self.histories[key][0] >= HISTORY_SECONDS] + if not due: + return + found = {} + if hasattr(self.ha, 'statistics'): + for hours in sorted({hours for _, hours in due}): + entities = {entity for entity, h in due if h == hours} + try: + found.update({(entity, hours): values for entity, values in (await self.ha.statistics(entities, hours)).items()}) + except Exception as error: + LOG.warning('Statistieken ophalen mislukt (%s); geschiedenis per sensor', type(error).__name__) + changed = set() + for key in due: + values = found.get(key) + if values is None: + try: + values = await self.ha.history(*key) + except Exception: + values = [] + old = self.histories.get(key) + self.histories[key] = (time.monotonic(), values) + if old is None or old[1] != values: + changed.add(key[0]) + if changed: + dirty = getattr(self.ha, 'dirty', None) + if dirty is not None: + dirty.update(changed) + self.ha.changed.set() + + async def history_loop(self): + """Background refresh of sensor histories, so the sync loop never waits for Home Assistant's database.""" + while True: + with contextlib.suppress(TimeoutError): + await asyncio.wait_for(self.history_wake.wait(), 30) + self.history_wake.clear() + if not self.ha.online or not hasattr(self.ha, 'history'): + continue + try: + await self.refresh_histories() + except Exception as error: + LOG.warning('Geschiedenis vernieuwen mislukt (%s)', type(error).__name__) def create_app(manager, development=False): csrf = secrets.token_urlsafe(32) @@ -475,9 +706,10 @@ def create_app(manager, development=False): async def index(request): return web.FileResponse(static / 'index.html') - def light_payload(): + def light_payload(screens=None): """Screens and update status: everything that changes while the page is open.""" - screens, entities = manager.inventory() + if screens is None: + screens = manager.screens() profiles = manager.firmware.profile_names() for screen in screens: screen['layout'] = manager.layouts.get(screen['id'], {'title': 'Thuis', 'tiles': []}) @@ -487,19 +719,21 @@ def create_app(manager, development=False): screen['dismiss_action'] = alert_service(screen.get('node'), 'dismiss_alert') return {'csrf': csrf, 'connected': manager.ha.online, 'screens': screens, 'pending': manager.pending_profiles(screens, profiles), - 'updates': manager.updates.summary(screens, profiles)}, entities + 'updates': manager.updates.summary(screens, profiles)} async def inventory(request): - payload, entities = light_payload() - if request.query.get('light') != '1': + if request.query.get('light') == '1': # The page polls the light form; entities, backgrounds and icons (~100 KB) only on demand. - payload['entities'] = entities - payload['backgrounds'] = TILE_BACKGROUNDS - payload['controls'] = controls_catalogue() - payload['icons'] = tile_icons.editor() - payload['alerts'] = alert_reference() - payload['builtin'] = [{'id': key, 'name': name, 'device': 'Ingebouwd op het scherm', 'area': '', 'state': 'ok'} for key, name in BUILTIN.items()] - payload['header'] = {**header_bar.catalogue(), 'suggestions': { - screen['id']: header_bar.suggestions(screen, entities, manager.ha.states, manager.registry_index()) for screen in payload['screens']}} + return web.json_response(light_payload()) + screens, entities = manager.inventory() + payload = light_payload(screens) + payload['entities'] = entities + payload['backgrounds'] = TILE_BACKGROUNDS + payload['controls'] = controls_catalogue() + payload['icons'] = tile_icons.editor() + payload['alerts'] = alert_reference() + payload['builtin'] = [{'id': key, 'name': name, 'device': 'Ingebouwd op het scherm', 'area': '', 'state': 'ok'} for key, name in BUILTIN.items()] + payload['header'] = {**header_bar.catalogue(), 'suggestions': { + screen['id']: header_bar.suggestions(screen, entities, manager.ha.states, manager.registry_index()) for screen in payload['screens']}} return web.json_response(payload) async def header_preview(request): """The top bar as a screen would draw it right now, so the editor shows unsaved changes live.""" @@ -521,7 +755,7 @@ def create_app(manager, development=False): with contextlib.suppress(TimeoutError): await asyncio.wait_for(wake.wait(), 3) wake.clear() - body = json.dumps(light_payload()[0], ensure_ascii=False) + body = json.dumps(light_payload(), ensure_ascii=False) if body != sent: await response.write(f'data: {body}\n\n'.encode()) sent = body @@ -545,9 +779,10 @@ def create_app(manager, development=False): return web.json_response(manager.updates.summary()) async def inspector(request): inbox=request.match_info['inbox'] - if inbox not in {s['id'] for s in manager.inventory()[0]}: raise ValueError('Onbekend scherm.') + screen=manager.screen(inbox) + if screen is None: raise ValueError('Onbekend scherm.') layout=manager.layouts.get(inbox,{'tiles':[]}) - return web.json_response({'screen':next(s for s in manager.inventory()[0] if s['id']==inbox), + return web.json_response({'screen':screen, 'delivery':manager.status.get(inbox), 'layout':layout, 'tiles':[{'entity':t['entity'],'state':manager.ha.states.get(t['entity'],{}).get('state'), 'attributes':state_message(i,t,manager.ha.states)['a'], @@ -592,7 +827,7 @@ async def main(): await runner.setup() await web.TCPSite(runner, '127.0.0.1' if development else '0.0.0.0', 8099).start() try: - await asyncio.gather(ha.run(), manager.run(), manager.updates.run()) + await asyncio.gather(ha.run(), manager.run(), manager.history_loop(), manager.updates.run()) finally: await runner.cleanup() diff --git a/screen_manager/app/updates.py b/screen_manager/app/updates.py index 85ee2d5..434509d 100644 --- a/screen_manager/app/updates.py +++ b/screen_manager/app/updates.py @@ -55,7 +55,7 @@ class Updater: return bool(self.task and not self.task.done()) def screen(self, inbox): - return next((s for s in self.manager.inventory()[0] if s['id'] == inbox), None) + return self.manager.screen(inbox) def resolve(self, screen, profiles=None): """Profile file and OTA address for a screen; None when the add-on cannot tell.""" @@ -89,7 +89,7 @@ class Updater: # Callers that already hold the inventory and profile list pass them in; the inventory # handler otherwise re-reads every ESPHome profile several times per request. if screens is None: - screens = self.manager.inventory()[0] + screens = self.manager.screens() if profiles is None: profiles = self.manager.firmware.profile_names() return [s['id'] for s in screens diff --git a/screen_manager/config.yaml b/screen_manager/config.yaml index ac8111a..156fc8a 100644 --- a/screen_manager/config.yaml +++ b/screen_manager/config.yaml @@ -1,5 +1,5 @@ name: ESP Screen Manager -version: "0.2.38" +version: "0.2.39" slug: esp_screen_manager description: Kies tegels voor je CYD en Guition touchscreen, zonder opnieuw flashen. url: https://github.com/MaxGramser/homeassistant_espscreen diff --git a/tests/test_alerts_reference.py b/tests/test_alerts_reference.py index 1421b95..420d864 100644 --- a/tests/test_alerts_reference.py +++ b/tests/test_alerts_reference.py @@ -82,7 +82,7 @@ class InventoryTests(unittest.IsolatedAsyncioTestCase): states = {'text.studio_1_tegelinstellingen': {'state': 'Ready'}, 'sensor.studio_1_apparaatnaam': {'state': 'studio-1'}, 'sensor.studio_1_schermfirmware': {'state': '0.2.31'}, 'text.oud_tegelinstellingen': {'state': 'Ready'}} changed = asyncio.Event() - async def send(self, inbox, message): pass + async def send(self, inbox, message, action=None): pass with tempfile.TemporaryDirectory() as temp: manager = Manager(HA(), Path(temp) / 'screens.json') async with TestClient(TestServer(create_app(manager, True))) as client: diff --git a/tests/test_header_bar.py b/tests/test_header_bar.py index 393ded1..ce5a7bc 100644 --- a/tests/test_header_bar.py +++ b/tests/test_header_bar.py @@ -244,7 +244,7 @@ class ManagerTests(unittest.IsolatedAsyncioTestCase): self.states = {'text.screen': state('Ready'), 'sensor.fw': state(firmware), 'light.a': state('on'), 'sensor.t': state('21.34', unit_of_measurement='°C', device_class='temperature'), 'binary_sensor.deur': state('off', device_class='door')} - async def send(self, inbox, message): + async def send(self, inbox, message, action=None): self.messages.append(message) return Manager(HA(), path) diff --git a/tests/test_portal.py b/tests/test_portal.py index 94276dc..d5a3d9c 100644 --- a/tests/test_portal.py +++ b/tests/test_portal.py @@ -81,7 +81,7 @@ class ManagerTests(unittest.IsolatedAsyncioTestCase): states={'text.screen':{'state':'Ready'},'light.a':{'state':'on','attributes':{'friendly_name':'Lamp'}}} changed=asyncio.Event() def __init__(self): self.messages=[] - async def send(self,inbox,message): self.messages.append((inbox,message)) + async def send(self,inbox,message,action=None): self.messages.append((inbox,message)) return Manager(HA(),path) async def test_update_restart_preserves_layout(self): @@ -136,7 +136,7 @@ class ManagerTests(unittest.IsolatedAsyncioTestCase): m=self.setup_manager(Path(temp)/'screens.json');layout={'title':'Thuis','tiles':[{'entity':'light.a','name':''}]} m.save('text.screen',layout) original=m.ha.send - async def interrupt(inbox,message): + async def interrupt(inbox,message,action=None): await original(inbox,message) m.save(inbox,{'title':'Leeg','tiles':[]}) m.ha.send=interrupt diff --git a/tests/test_scaling.py b/tests/test_scaling.py new file mode 100644 index 0000000..fb16d0b --- /dev/null +++ b/tests/test_scaling.py @@ -0,0 +1,323 @@ +"""Daily efficiency with many screens: incremental sync, one action per message, the keepalive ping, +bundled history in the background and the cached screen list (app 0.2.39 / firmware 0.2.33).""" +import asyncio +import importlib.util +import json +from pathlib import Path +import sys +import tempfile +import time +import unittest +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'screen_manager/app')) +from core import TRANSPORT_MIN_FIRMWARE, encode, message_action, packets, revision + +HAS_AIOHTTP = importlib.util.find_spec('aiohttp') is not None +if HAS_AIOHTTP: + import server + from server import FULL_REPEAT_SECONDS, HISTORY_SECONDS, KEEPALIVE_SECONDS, HomeAssistant, Manager, samples + + +class ProtocolTests(unittest.TestCase): + def test_encode_revision_and_action(self): + message = {'v': 1, 'op': 'layout', 'title': 'Kantoor 1', 'entities': ['light.a']} + self.assertEqual(json.loads(encode(message)), message) + self.assertNotIn(', ', encode(message), 'compact separators') + with self.assertRaises(ValueError): + encode({'x': 'y' * 4096}) + # The chunks carry exactly the encoded message. + import base64 + self.assertEqual(base64.b64decode(''.join(p.split('|')[3] for p in packets(message))).decode(), encode(message)) + # Same content, any key order: same revision; any change: another one. + self.assertEqual(revision(message), revision({'entities': ['light.a'], 'title': 'Kantoor 1', 'op': 'layout', 'v': 1})) + self.assertNotEqual(revision(message), revision({**message, 'title': 'Kantoor 2'})) + self.assertEqual(len(revision(message)), 12) + self.assertEqual(message_action('kantoor-1'), 'esphome.kantoor_1_screen_message') + self.assertIsNone(message_action(None)) + self.assertEqual(TRANSPORT_MIN_FIRMWARE, (0, 2, 33)) + + def test_samples_take_the_last_value_per_bucket(self): + events = [(100, 1.0), (50, 2.0), (1000, 3.0)] + out = samples(events, 0, 2400) + self.assertEqual(out[0], 1.0, 'both early events fall in the first bucket; the later one wins') + self.assertEqual(out[8], 1.0) + self.assertEqual(out[9], 3.0) + self.assertEqual(out[23], 3.0, 'carried forward') + self.assertEqual(samples([], 0, 2400), [None] * 24) + + +def fake_ha(firmware='0.2.33', node='kantoor-1'): + class HA: + online = True + + def __init__(self): + self.registry = [{'entity_id': 'text.screen', 'platform': 'esphome', 'original_name': 'Tegelinstellingen', 'device_id': 'd1'}, + {'entity_id': 'sensor.fw', 'platform': 'esphome', 'original_name': 'Schermfirmware', 'device_id': 'd1'}, + {'entity_id': 'text.node', 'platform': 'esphome', 'original_name': 'Apparaatnaam', 'device_id': 'd1'}, + {'entity_id': 'light.a', 'platform': 'hue'}, {'entity_id': 'light.b', 'platform': 'hue'}, + {'entity_id': 'sensor.t', 'platform': 'mqtt'}, {'entity_id': 'sensor.co2', 'platform': 'mqtt'}] + self.devices, self.areas = [{'id': 'd1', 'name': 'Kantoor 1'}], [] + self.states = {'text.screen': {'state': 'Gesynchroniseerd'}, 'sensor.fw': {'state': firmware}, 'text.node': {'state': node}, + 'light.a': {'state': 'on', 'attributes': {'friendly_name': 'Lamp A'}}, + 'light.b': {'state': 'off', 'attributes': {'friendly_name': 'Lamp B'}}, + 'sensor.t': {'state': '21.5', 'attributes': {'unit_of_measurement': '°C', 'friendly_name': 'Temperatuur'}}, + 'sensor.co2': {'state': '600', 'attributes': {'unit_of_measurement': 'ppm', 'friendly_name': 'CO2'}}} + self.changed = asyncio.Event() + self.dirty = set() + self.relevant = None + self.messages = [] + self.stat_calls, self.history_calls = [], [] + self.stats = {} + + async def send(self, inbox, message, action=None): + self.messages.append((inbox, message, action)) + + async def statistics(self, entities, hours): + self.stat_calls.append((sorted(entities), hours)) + return {e: self.stats[e] for e in entities if e in self.stats} + + async def history(self, entity, hours): + self.history_calls.append((entity, hours)) + return [1.0] * 24 + return HA() + + +@unittest.skipUnless(HAS_AIOHTTP, 'Run using .venv-portal/bin/python for server tests') +class IncrementalSync(unittest.IsolatedAsyncioTestCase): + def manager(self, tmp, **kw): + m = Manager(fake_ha(**kw), Path(tmp) / 'screens.json') + m.save('text.screen', {'title': 'Kantoor 1', 'tiles': [{'entity': 'light.a', 'name': ''}, {'entity': 'light.b', 'name': ''}], + 'header': {'items': [{'type': 'clock'}, {'type': 'entity', 'entity': 'sensor.t', 'content': 'state', 'icon': 'auto', 'show': 'always'}]}}) + return m + + async def test_only_dirty_tiles_are_rebuilt_and_sent(self): + with tempfile.TemporaryDirectory() as tmp: + m = self.manager(tmp) + layout = m.layouts['text.screen'] + self.assertTrue(await m.sync_one('text.screen', layout, dirty=set())) + ops = [msg['op'] for _, msg, _ in m.ha.messages] + self.assertEqual(ops, ['layout', 'header', 'state', 'state']) + self.assertIn('rev', m.ha.messages[0][1]) + self.assertTrue(all(action == 'esphome.kantoor_1_screen_message' for _, _, action in m.ha.messages), 'firmware 0.2.33: one action per message') + m.ha.messages.clear() + # Both lamps change in HA, but only light.a is reported dirty: light.b's message is reused as sent. + m.ha.states['light.a']['state'] = 'off' + m.ha.states['light.b']['state'] = 'on' + with patch.object(server, 'state_message', wraps=server.state_message) as built: + self.assertTrue(await m.sync_one('text.screen', layout, dirty={'light.a'})) + self.assertEqual(built.call_count, 1, 'one tile rebuilt') + self.assertEqual([(msg['op'], msg.get('entity')) for _, msg, _ in m.ha.messages], [('state', 'light.a')]) + m.ha.messages.clear() + # Nothing dirty: nothing rebuilt, nothing sent, and the top bar untouched. + with patch.object(server.header_bar, 'message', wraps=server.header_bar.message) as bar: + self.assertFalse(await m.sync_one('text.screen', layout, dirty=set())) + self.assertEqual(bar.call_count, 0) + self.assertEqual(m.ha.messages, []) + # The top bar follows its own entity. + m.ha.states['sensor.t']['state'] = '22.0' + self.assertTrue(await m.sync_one('text.screen', layout, dirty={'sensor.t'})) + self.assertEqual([msg['op'] for _, msg, _ in m.ha.messages], ['header']) + m.ha.messages.clear() + # None rebuilds everything and sends the differences (light.b was still stale). + self.assertTrue(await m.sync_one('text.screen', layout, dirty=None)) + self.assertEqual([(msg['op'], msg.get('entity')) for _, msg, _ in m.ha.messages], [('state', 'light.b')]) + m.ha.messages.clear() + # force repeats everything, as the hourly safety net and older firmware need. + self.assertTrue(await m.sync_one('text.screen', layout, force=True, dirty=set())) + self.assertEqual([msg['op'] for _, msg, _ in m.ha.messages], ['layout', 'header', 'state', 'state']) + + async def test_ping_and_old_firmware_transport(self): + with tempfile.TemporaryDirectory() as tmp: + m = self.manager(tmp) + layout = m.layouts['text.screen'] + await m.sync_one('text.screen', layout, dirty=set()) + rev = m.ha.messages[0][1]['rev'] + m.ha.messages.clear() + await m.ping('text.screen', m.screen('text.screen')) + self.assertEqual(m.ha.messages, [('text.screen', {'v': 1, 'op': 'ping', 'rev': rev, 'keepalive': KEEPALIVE_SECONDS}, 'esphome.kantoor_1_screen_message')]) + self.assertEqual(rev, revision({k: v for k, v in m.sent['text.screen']['layout'].items() if k != 'rev'})) + # Firmware before 0.2.33: base64 chunks in the text inbox, no ping. + old = self.manager(tmp, firmware='0.2.32') + self.assertIsNone(old.transport('text.screen')) + self.assertFalse(old.supports_ping('text.screen')) + await old.sync_one('text.screen', old.layouts['text.screen'], dirty=set()) + self.assertTrue(all(action is None for _, _, action in old.ha.messages)) + old.ha.messages.clear() + await old.ping('text.screen', old.screen('text.screen')) # harmless: nothing recorded as sent by ping + self.assertEqual([msg['op'] for _, msg, _ in old.ha.messages], ['ping'], 'ping itself is version-agnostic; run() decides') + + async def run_loop_for(self, m, seconds): + task = asyncio.create_task(m.run()) + await asyncio.sleep(seconds) + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + + async def test_run_loop_pings_resends_and_repeats(self): + with tempfile.TemporaryDirectory() as tmp: + m = self.manager(tmp) + m.ha.changed.set() + await self.run_loop_for(m, 0.6) + ops = [msg['op'] for _, msg, _ in m.ha.messages] + self.assertEqual(ops, ['layout', 'header', 'state', 'state'], 'first pass sends everything') + self.assertEqual(m.ha.relevant, {'light.a', 'light.b', 'sensor.t', 'text.screen', 'sensor.fw', 'text.node'}) + m.ha.messages.clear() + # A wake without dirty entities sends nothing, not even a ping before its time. + m.ha.changed.set() + await self.run_loop_for(m, 0.6) + self.assertEqual(m.ha.messages, []) + # Two minutes later: only the ping. + m.pinged['text.screen'] = time.monotonic() - KEEPALIVE_SECONDS + m.ha.changed.set() + await self.run_loop_for(m, 0.6) + self.assertEqual([msg['op'] for _, msg, _ in m.ha.messages], ['ping']) + m.ha.messages.clear() + # The screen answered a ping with a mismatch (or restarted): everything again, once. + m.ha.states['text.screen'] = {'state': 'Indeling opnieuw nodig'} + m.last['text.screen'] = time.monotonic() - 130 + m.ha.changed.set() + await self.run_loop_for(m, 0.6) + self.assertEqual([msg['op'] for _, msg, _ in m.ha.messages], ['layout', 'header', 'state', 'state']) + m.ha.messages.clear() + # The same answer right after a full send does not loop. + m.ha.changed.set() + await self.run_loop_for(m, 0.6) + self.assertEqual(m.ha.messages, []) + # The hourly safety net repeats everything. + m.ha.states['text.screen'] = {'state': 'Gesynchroniseerd'} + m.last['text.screen'] = time.monotonic() - FULL_REPEAT_SECONDS + m.ha.changed.set() + await self.run_loop_for(m, 0.6) + self.assertEqual([msg['op'] for _, msg, _ in m.ha.messages], ['layout', 'header', 'state', 'state']) + m.ha.messages.clear() + # A registry refresh rebuilds everything without differences: the ping is not postponed. + m.ha.messages.clear() + m.pinged['text.screen'] = time.monotonic() - KEEPALIVE_SECONDS + m.ha.registry = list(m.ha.registry) + m.ha.changed.set() + await self.run_loop_for(m, 0.6) + self.assertEqual([msg['op'] for _, msg, _ in m.ha.messages], ['ping']) + m.ha.messages.clear() + # A dirty state change sends that tile only. + m.ha.states['light.b']['state'] = 'on' + m.ha.dirty.add('light.b') + m.ha.changed.set() + await self.run_loop_for(m, 0.6) + self.assertEqual([(msg['op'], msg.get('entity')) for _, msg, _ in m.ha.messages], [('state', 'light.b')]) + + async def test_old_firmware_keeps_the_full_keepalive(self): + with tempfile.TemporaryDirectory() as tmp: + m = self.manager(tmp, firmware='0.2.32') + m.ha.changed.set() + await self.run_loop_for(m, 0.6) + m.ha.messages.clear() + m.last['text.screen'] = time.monotonic() - KEEPALIVE_SECONDS + m.ha.changed.set() + await self.run_loop_for(m, 0.6) + self.assertEqual([msg['op'] for _, msg, _ in m.ha.messages], ['layout', 'header', 'state', 'state']) + self.assertTrue(all(action is None for _, _, action in m.ha.messages)) + + +@unittest.skipUnless(HAS_AIOHTTP, 'Run using .venv-portal/bin/python for server tests') +class ScreenCache(unittest.IsolatedAsyncioTestCase): + async def test_screens_are_rediscovered_only_when_their_inputs_change(self): + with tempfile.TemporaryDirectory() as tmp: + m = Manager(fake_ha(), Path(tmp) / 'screens.json') + with patch.object(server, 'discover_screens', wraps=server.discover_screens) as found: + first = m.screens() + self.assertEqual([s['id'] for s in first], ['text.screen']) + first[0]['layout'] = 'mutated by a caller' + self.assertNotIn('layout', m.screens()[0], 'callers get copies') + m.ha.states['light.a']['state'] = 'off' # not a screen diagnostic + m.screens() + self.assertEqual(found.call_count, 1) + m.ha.states['sensor.fw'] = {'state': '0.2.34'} + self.assertEqual(m.screens()[0]['firmware'], '0.2.34') + self.assertEqual(found.call_count, 2) + m.ha.registry = list(m.ha.registry) # a new registry object from HA + m.screens() + self.assertEqual(found.call_count, 3) + self.assertEqual(found.call_args[0][0], m.screen_registry(), 'only the screens\' own entries are walked') + # The watched set follows layouts and registry, not every call. + with patch.object(server, 'header_items', wraps=server.header_items) as items: + m.watched_entities(); m.watched_entities() + self.assertEqual(items.call_count, 0, 'no layouts yet: nothing to walk') + m.save('text.screen', {'title': 'K', 'tiles': [{'entity': 'light.a', 'name': ''}]}) + self.assertIn('light.a', m.watched_entities()) + self.assertIn('text.screen', m.watched_entities()) + calls = items.call_count + m.watched_entities() + self.assertEqual(items.call_count, calls) + + +@unittest.skipUnless(HAS_AIOHTTP, 'Run using .venv-portal/bin/python for server tests') +class HistoryInTheBackground(unittest.IsolatedAsyncioTestCase): + async def test_statistics_bundle_rest_fallback_and_dirty_marking(self): + with tempfile.TemporaryDirectory() as tmp: + m = Manager(fake_ha(), Path(tmp) / 'screens.json') + m.ha.stats = {'sensor.t': [21.0] * 24} + m.save('text.screen', {'title': 'K', 'tiles': [{'entity': 'sensor.t', 'name': '', 'options': {'history_hours': 24}}, + {'entity': 'sensor.co2', 'name': '', 'options': {'history_hours': 6}}, + {'entity': 'light.a', 'name': ''}]}) + self.assertTrue(m.history_wake.is_set(), 'a save wakes the history task') + # The sync loop never waits for history: the first state message goes out without it. + await m.sync_one('text.screen', m.layouts['text.screen'], dirty=set()) + self.assertNotIn('history', m.ha.messages[1][1]) + m.ha.messages.clear() + await m.refresh_histories() + self.assertEqual(m.ha.stat_calls, [(['sensor.co2'], 6), (['sensor.t'], 24)], 'one statistics request per window') + self.assertEqual(m.ha.history_calls, [('sensor.co2', 6)], 'REST only for the sensor without statistics') + self.assertEqual(m.ha.dirty, {'sensor.t', 'sensor.co2'}) + self.assertTrue(m.ha.changed.is_set()) + # The next pass rebuilds just those tiles, now with history. + dirty, m.ha.dirty = m.ha.dirty, set() + await m.sync_one('text.screen', m.layouts['text.screen'], dirty=dirty) + sent = {msg['entity']: msg for _, msg, _ in m.ha.messages} + self.assertEqual(sent['sensor.t']['history'], {'hours': 24, 'values': [21.0] * 24}) + self.assertEqual(sent['sensor.co2']['history'], {'hours': 6, 'values': [1.0] * 24}) + self.assertNotIn('light.a', sent) + # Fresh entries are not fetched again; unchanged values mark nothing dirty. + m.ha.stat_calls.clear(); m.ha.history_calls.clear() + await m.refresh_histories() + self.assertEqual((m.ha.stat_calls, m.ha.history_calls, m.ha.dirty), ([], [], set())) + for key in m.histories: + m.histories[key] = (time.monotonic() - HISTORY_SECONDS, m.histories[key][1]) + await m.refresh_histories() + self.assertEqual(len(m.ha.stat_calls), 2) + self.assertEqual(m.ha.dirty, set(), 'same values: no resend') + # A tile that left the layout drops out of the cache. + m.save('text.screen', {'title': 'K', 'tiles': [{'entity': 'sensor.t', 'name': ''}]}) + await m.refresh_histories() + self.assertEqual(set(m.histories), {('sensor.t', 24)}) + + async def test_statistics_request_and_bucketing(self): + ha = HomeAssistant(None, 'http://ha/api', 'token') + now = time.time() + calls = [] + async def request(kind, **data): + calls.append((kind, data)) + # Hourly rows in milliseconds (HA 2023.9+); the newest hour has no row yet. + return {'sensor.t': [{'start': (now - 24 * 3600 + h * 3600) * 1000, 'mean': 20 + h} for h in range(23)], + 'sensor.e': [{'start': (now - 3600) * 1000, 'state': 12.5, 'sum': 100}], + 'sensor.other': [{'start': 1, 'mean': 1}]} + ha.request = request + found = await ha.statistics({'sensor.t', 'sensor.e', 'sensor.none'}, 24) + self.assertEqual(calls[0][0], 'recorder/statistics_during_period') + self.assertEqual(calls[0][1]['period'], 'hour') + self.assertEqual(calls[0][1]['statistic_ids'], ['sensor.e', 'sensor.none', 'sensor.t']) + self.assertEqual(calls[0][1]['types'], ['mean', 'state']) + self.assertEqual(set(found), {'sensor.t', 'sensor.e'}, 'no statistics: absent, so the REST history takes over') + # A row that starts a hair before a bucket's end is that bucket's last value. + self.assertEqual(found['sensor.t'][:3], [21, 22, 23]) + self.assertEqual(found['sensor.t'][23], 42, 'the running hour carries the previous mean forward') + self.assertEqual(found['sensor.e'][23], 12.5, 'sum sensors give their state') + self.assertIsNone(found['sensor.e'][0]) + await ha.statistics({'sensor.t'}, 6) + self.assertEqual(calls[1][1]['period'], '5minute') + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_updates.py b/tests/test_updates.py index 10f81e2..02fb45d 100644 --- a/tests/test_updates.py +++ b/tests/test_updates.py @@ -137,7 +137,7 @@ class UpdaterTests(unittest.IsolatedAsyncioTestCase): 'text.fw1': {'state': '0.2.16'}, 'text.screen2': {'state': 'Ready'}, 'text.fw2': {'state': '0.2.16'}} changed = asyncio.Event() def __init__(self): self.messages, self.calls = [], [] - async def send(self, inbox, message): self.messages.append((inbox, message)) + async def send(self, inbox, message, action=None): self.messages.append((inbox, message)) async def request(self, kind, **data): self.calls.append((kind, data)) manager = Manager(HA(), Path(path) / 'screens.json') manager.firmware = FakeFirmware(manager.ha, outcome)