Cache ESPHome profile metadata and stop polling hidden tabs (0.2.19)

/api/inventory re-read every ESPHome profile with pure-Python PyYAML up to
four times per request, which blocked the event loop for seconds on a Pi.
Profiles are now cached per file on (inode, mtime_ns, size), parsed with
libyaml when available, and read once per request. The sync loop computes
the inventory once per iteration. The page pauses polling while hidden.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

screen_manager/CHANGELOG.md

@@ -1,3 +1,10 @@
+## 0.2.19 (firmware 0.2.18 blijft actueel)
+
+- **Sneller overzicht**: `/api/inventory` las bij elke aanroep alle ESPHome-profielen opnieuw in (tot vier keer per verzoek), waardoor de add-on op een Pi seconden per verzoek blokkeerde. Profielen worden nu per bestand gecachet op inode, wijzigingstijd en grootte; alleen een gewijzigd profiel wordt opnieuw gelezen en verwijderde profielen verdwijnen direct. Het parsen zelf gebruikt libyaml wanneer die beschikbaar is (circa tien keer sneller).
+- Het overzicht en de updatestatus lezen de profiellijst en de HA-inventaris nog één keer per verzoek; de synchronisatielus rekent de inventaris één keer per ronde uit in plaats van per scherm.
+- De pagina pollt niet meer in een verborgen tabblad en ververst direct zodra het tabblad weer zichtbaar is.
+- Alleen de app verandert; firmware 0.2.18 blijft actueel.
+
 ## 0.2.18 (firmware 0.2.18)
 
 - **Eigen icoon per tegel**: in het tegelpaneel staat onder de naam het veld **Icoon**. Kies uit 150 iconen in twaalf groepen (verlichting, ruimtes, klimaat, weer, media en muziek, beveiliging, apparaten, energie, zonwering, tuin en huisdieren, mensen en onderweg, overig), met zoeken op Nederlandse naam, Engelse MDI-naam of groep. De mockup en de kop van het paneel tonen het gekozen icoon direct.

screen_manager/app/firmware.py

@@ -12,10 +12,24 @@ import time
 import yaml
 from core import installation_yaml
 
-class LenientLoader(yaml.SafeLoader):
+# libyaml parses a 300 KB profile roughly ten times faster than the pure-Python loader.
+class LenientLoader(yaml.CSafeLoader if getattr(yaml, '__with_libyaml__', False) else yaml.SafeLoader):
     """Reads profile metadata without resolving !secret or !include."""
 LenientLoader.add_multi_constructor('!', lambda loader, suffix, node: None)
 
+def profile_meta(text):
+    """{'node', 'friendly'} from profile YAML, or None when it has no esphome block."""
+    data = yaml.load(text, Loader=LenientLoader)
+    block = data.get('esphome') if isinstance(data, dict) else None
+    if not isinstance(block, dict):
+        return None
+    substitutions = data.get('substitutions') if isinstance(data.get('substitutions'), dict) else {}
+    def resolve(value):
+        if isinstance(value, str) and value.startswith('${') and value.endswith('}'):
+            value = substitutions.get(value[2:-1])
+        return value if isinstance(value, str) else None
+    return {'node': resolve(block.get('name')), 'friendly': resolve(block.get('friendly_name'))}
+
 class Firmware:
     def __init__(self, root, data):
         self.root, self.data = Path(root).resolve(), Path(data)
@@ -23,6 +37,7 @@ class Firmware:
         self.task = None
         self.logs = deque(maxlen=300)
         self.process = None
+        self._names = {}  # file -> (stat signature, meta or None); parsed only when the file changes
 
     def profiles(self):
         if not self.root.exists(): return []
@@ -30,22 +45,28 @@ class Firmware:
                 if p.name != 'secrets.yaml' and p.is_file() and not p.is_symlink()]
 
     def profile_names(self):
-        """{file: {'node': esphome name, 'friendly': friendly name}} for every readable profile."""
+        """{file: {'node': esphome name, 'friendly': friendly name}} for every readable profile.
-        found = {}
+
+        Parsing is cached per file on inode, mtime and size, so a request only pays for changed
+        profiles. The dict is rebuilt from the current directory listing, so deleted files drop out.
+        """
+        found, cache = {}, {}
         for entry in self.profiles():
+            path = self.root / entry['file']
             try:
-                data = yaml.load((self.root / entry['file']).read_text(), Loader=LenientLoader)
+                st = path.stat()
+                key = (st.st_ino, st.st_mtime_ns, st.st_size)
+                cached = self._names.get(entry['file'])
+                if cached and cached[0] == key:
+                    meta = cached[1]
+                else:
+                    meta = profile_meta(path.read_text())
             except (OSError, yaml.YAMLError, UnicodeError):
                 continue
-            block = data.get('esphome') if isinstance(data, dict) else None
+            cache[entry['file']] = (key, meta)
-            if not isinstance(block, dict):
+            if meta:
-                continue
+                found[entry['file']] = meta
-            substitutions = data.get('substitutions') if isinstance(data.get('substitutions'), dict) else {}
+        self._names = cache
-            def resolve(value):
-                if isinstance(value, str) and value.startswith('${') and value.endswith('}'):
-                    value = substitutions.get(value[2:-1])
-                return value if isinstance(value, str) else None
-            found[entry['file']] = {'node': resolve(block.get('name')), 'friendly': resolve(block.get('friendly_name'))}
         return found
 
     def ports(self):

screen_manager/app/server.py

@@ -182,7 +182,8 @@ class Manager:
     def inventory(self):
         return discover(self.ha.registry, self.ha.states, self.ha.devices, self.ha.areas)
 
-    def firmware_version(self, inbox):
+    def firmware_version(self, inbox, screen=None):
+        if screen is None:
             screen=next((s for s in self.inventory()[0] if s['id']==inbox), {})
         try:
             version=tuple(int(part) for part in screen.get('firmware','').split('.'))
@@ -194,18 +195,20 @@ class Manager:
         version=self.firmware_version(inbox)
         return bool(version) and version >= (0,2,7)
 
-    def needs_firmware(self, inbox, layout):
+    def needs_firmware(self, inbox, layout, screen=None):
         """Version string the screen must run first, or None when the layout can be sent."""
         needed=min_firmware(layout)
-        if needed and not ((self.firmware_version(inbox) or (0,0,0)) >= needed):
+        if needed and not ((self.firmware_version(inbox, screen) or (0,0,0)) >= needed):
             return '.'.join(str(part) for part in needed)
         return None
 
     def save(self, inbox, data):
-        if inbox not in {s['id'] for s in self.inventory()[0]}:
+        screens, entities = self.inventory()
+        screen=next((s for s in screens if s['id']==inbox), None)
+        if screen is None:
             raise ValueError('Dit is geen gekoppeld ESP-scherm. Vernieuw het overzicht.')
         layout = validate_layout(data)
-        needed = self.needs_firmware(inbox, layout)
+        needed = self.needs_firmware(inbox, layout, screen)
         if needed:
             raise ValueError(f"Installeer eerst schermfirmware {needed} of nieuwer voor deze tegels.")
         # A still-open older UI may save tiles without the new optional settings.
@@ -215,7 +218,6 @@ class Manager:
             layout['settings']['swipe_pages']=self.layouts.get(inbox,{}).get('settings',{}).get('swipe_pages',False)
         if 'settings' in layout and 'rotation' not in data.get('settings',{}):
             layout['settings']['rotation']=self.layouts.get(inbox,{}).get('settings',{}).get('rotation',0)
-        screen=next(s for s in self.inventory()[0] if s['id']==inbox)
         if layout.get('settings',{}).get('rotation',0) and screen.get('board')!='guition':
             raise ValueError('Rotatie vereist een Guition met firmware 0.2.9 of nieuwer.')
         old_tiles = {t['entity']:t for t in self.layouts.get(inbox,{}).get('tiles',[])}
@@ -226,7 +228,7 @@ class Manager:
                     tile['options'][key]=old_options[key]
             if 'options' not in tile and 'options' in old_tiles.get(tile['entity'],{}):
                 tile['options'] = old_tiles[tile['entity']]['options'].copy()
-        known = {e['id'] for e in self.inventory()[1]} | set(BUILTIN)
+        known = {e['id'] for e in entities} | set(BUILTIN)
         if any(t['entity'] not in known for t in layout['tiles']):
             raise ValueError('Een gekozen entiteit bestaat niet meer. Zoek de nieuwe entiteit op.')
         updated = {**self.layouts, inbox: layout}
@@ -251,8 +253,10 @@ class Manager:
             entry=(time.monotonic(),value);store[key]=entry
         return entry[1]
 
-    async def sync_one(self, inbox, layout, force=False):
+    async def sync_one(self, inbox, layout, force=False, screen=None):
-        needed = self.needs_firmware(inbox, layout)
+        if screen is None:
+            screen=next((s for s in self.inventory()[0] if s['id']==inbox), {})
+        needed = self.needs_firmware(inbox, layout, screen)
         if needed:
             self.status[inbox]=f"Indeling bewaard; firmware {needed}+ nodig voor deze tegels"
             return
@@ -260,7 +264,7 @@ class Manager:
         if 'settings' in layout:
             messages[0]['settings'] = {k:v for k,v in layout['settings'].items() if k not in ('swipe_pages','rotation')}
             messages[0]['swipe_pages'] = layout['settings'].get('swipe_pages',False)
-            if next((s.get('board') for s in self.inventory()[0] if s['id']==inbox),None)=='guition':
+            if screen.get('board')=='guition':
                 messages[0]['rotation'] = layout['settings'].get('rotation',0)
         for i,tile in enumerate(layout['tiles']):
             forecast=None
@@ -321,7 +325,7 @@ class Manager:
                     force = time.monotonic() - self.last.get(inbox, 0) >= 25
                     # last records full keepalive, not unrelated HA state events.
                     before = self.last.get(inbox, 0)
-                    await self.sync_one(inbox, self.layouts[inbox], force)
+                    await self.sync_one(inbox, self.layouts[inbox], force, screen)
                     if not force:
                         self.last[inbox] = before
                 except Exception as error:
@@ -357,13 +361,14 @@ def create_app(manager, development=False):
         return web.FileResponse(static / 'index.html')
     async def inventory(request):
         screens, entities = manager.inventory()
+        profiles = manager.firmware.profile_names()
         for screen in screens:
             screen['layout'] = manager.layouts.get(screen['id'], {'title': 'Thuis', 'tiles': []})
             screen['delivery'] = manager.status.get(screen['id'], 'Kies je eerste tegels')
-            screen['update'] = manager.updates.state_for(screen)
+            screen['update'] = manager.updates.state_for(screen, profiles)
         builtin = [{'id': key, 'name': name, 'device': 'Ingebouwd op het scherm', 'area': '', 'state': 'ok'} for key, name in BUILTIN.items()]
         return web.json_response({'csrf': csrf, 'connected': manager.ha.online, 'screens': screens, 'entities': entities,
-                                  'backgrounds': TILE_BACKGROUNDS, 'icons': tile_icons.editor(), 'builtin': builtin, 'updates': manager.updates.summary()})
+                                  'backgrounds': TILE_BACKGROUNDS, 'icons': tile_icons.editor(), 'builtin': builtin, 'updates': manager.updates.summary(screens, profiles)})
     async def save(request):
         manager.save(request.match_info['inbox'], await request.json())
         return web.json_response({'saved': True})

screen_manager/app/static/app.js

@@ -1014,12 +1014,21 @@ window.addEventListener("beforeunload", (e) => {
   }
 });
 refresh();
-(function poll() {
+// Poll only while the tab is visible; a hidden tab would otherwise keep the add-on busy.
-  setTimeout(async () => {
+let pollTimer;
-    await refresh();
+function poll() {
+  clearTimeout(pollTimer);
+  pollTimer = setTimeout(async () => {
+    if (!document.hidden) await refresh();
     poll();
   }, inventory.updates?.busy ? 3000 : 10000);
-})();
+}
+poll();
+document.addEventListener("visibilitychange", async () => {
+  if (document.hidden) return;
+  await refresh();
+  poll();
+});
 
 // Shared firmware workspace; always select a concrete profile and upload target.
 let firmwarePoll;

screen_manager/app/updates.py

@@ -57,8 +57,9 @@ class Updater:
     def screen(self, inbox):
         return next((s for s in self.manager.inventory()[0] if s['id'] == inbox), None)
 
-    def resolve(self, screen):
+    def resolve(self, screen, profiles=None):
         """Profile file and OTA address for a screen; None when the add-on cannot tell."""
+        if profiles is None:
             profiles = self.manager.firmware.profile_names()
         profile = next((f for f, p in profiles.items() if screen.get('node') and p['node'] == screen['node']), None)
         if profile is None and screen.get('device'):
@@ -67,9 +68,9 @@ class Updater:
             profile = matches[0] if len(matches) == 1 else None
         return profile, screen.get('ip') or self.hosts.get(screen['id'])
 
-    def state_for(self, screen):
+    def state_for(self, screen, profiles=None):
         version = parse_version(screen.get('firmware'))
-        profile, host = self.resolve(screen)
+        profile, host = self.resolve(screen, profiles)
         if screen['id'] == self.current:
             state = 'running'
         elif screen['id'] in self.queue:
@@ -80,13 +81,19 @@ class Updater:
                 'profile': profile, 'host': host, 'state': state, 'phase': self.phase if state == 'running' else None,
                 'result': self.results.get(screen['id'])}
 
-    def summary(self):
+    def summary(self, screens=None, profiles=None):
         return {'auto': self.auto, 'target': FIRMWARE_VERSION, 'busy': self.current,
-                'pending': len(self.pending()), 'last_round': self.last_round}
+                'pending': len(self.pending(screens, profiles)), 'last_round': self.last_round}
 
-    def pending(self):
+    def pending(self, screens=None, profiles=None):
-        return [s['id'] for s in self.manager.inventory()[0]
+        # Callers that already hold the inventory and profile list pass them in; the inventory
-                if s['online'] and self.state_for(s)['available'] and all(self.resolve(s))]
+        # handler otherwise re-reads every ESPHome profile several times per request.
+        if screens is None:
+            screens = self.manager.inventory()[0]
+        if profiles is None:
+            profiles = self.manager.firmware.profile_names()
+        return [s['id'] for s in screens
+                if s['online'] and self.state_for(s, profiles)['available'] and all(self.resolve(s, profiles))]
 
     def set_auto(self, enabled):
         if not isinstance(enabled, bool):

screen_manager/config.yaml

@@ -1,5 +1,5 @@
 name: ESP Screen Manager
-version: "0.2.18"
+version: "0.2.19"
 slug: esp_screen_manager
 description: Kies tegels voor je CYD en Guition touchscreen, zonder opnieuw flashen.
 url: https://github.com/MaxGramser/homeassistant_espscreen

tests/test_updates.py

@@ -3,6 +3,7 @@ import asyncio
 from datetime import datetime, timezone
 import importlib.util
 import json
+import os
 from pathlib import Path
 import re
 import sys
@@ -12,6 +13,7 @@ import unittest
 ROOT = Path(__file__).resolve().parents[1]
 sys.path.insert(0, str(ROOT / 'screen_manager/app'))
 from core import FIRMWARE_VERSION, discover
+import firmware
 from firmware import Firmware
 import updates
 from updates import Updater, parse_version
@@ -64,6 +66,31 @@ class ProfileNameTests(unittest.TestCase):
             self.assertNotIn('broken.yaml', names)
             self.assertNotIn('secrets.yaml', names)
 
+    def test_profiles_are_parsed_only_when_changed(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            f = Firmware(tmp, tmp)
+            a, b = Path(tmp) / 'a.yaml', Path(tmp) / 'b.yaml'
+            a.write_text('esphome:\n  name: a\n  friendly_name: A\n')
+            b.write_text('esphome:\n  name: b\n  friendly_name: B\n')
+            parsed = []
+            original = firmware.profile_meta
+            firmware.profile_meta = lambda text: parsed.append(text) or original(text)
+            try:
+                self.assertEqual(set(f.profile_names()), {'a.yaml', 'b.yaml'})
+                self.assertEqual(len(parsed), 2)
+                self.assertEqual(set(f.profile_names()), {'a.yaml', 'b.yaml'})
+                self.assertEqual(len(parsed), 2, 'unchanged files must not be parsed again')
+                b.write_text('esphome:\n  name: b2\n  friendly_name: B2\n')
+                os.utime(b, ns=(b.stat().st_atime_ns, b.stat().st_mtime_ns + 1_000_000))
+                names = f.profile_names()
+                self.assertEqual(len(parsed), 3, 'only the changed file is parsed')
+                self.assertEqual(names['b.yaml'], {'node': 'b2', 'friendly': 'B2'})
+                a.unlink()
+                self.assertEqual(set(f.profile_names()), {'b.yaml'})
+                self.assertEqual(len(parsed), 3)
+            finally:
+                firmware.profile_meta = original
+
 
 class FakeFirmware:
     """Stands in for the ESPHome CLI: records jobs and flips the screen to the target version."""