Per-pool interval watermarks, and cpu load repairs

src/urt/driver/bk7231/alloc.d

@@ -10,6 +10,7 @@ enum has_memsize  = false;
 enum has_exec     = false;
 enum has_retain   = false;
 enum has_memflags = false;
+enum has_pool_usage = false;
 
 void[] _alloc(size_t size, size_t alignment, MemFlags) pure
 {

src/urt/driver/bl_common/alloc.d

@@ -49,6 +49,7 @@ version (BouffaloUnifiedAlloc):
 
 import urt.attribute : fast_data;
 import urt.mem.alloc : MemFlags;
+import urt.mem.pressure : note_pool_usage;
 
 @nogc nothrow:
 
@@ -59,6 +60,7 @@ enum has_memsize  = true;
 enum has_exec     = false;
 enum has_retain   = false;
 enum has_memflags = true;
+enum has_pool_usage = true;
 
 
 void[] _alloc(size_t size, size_t alignment, MemFlags flags) pure
@@ -399,6 +401,7 @@ void[] alloc_impl(size_t size, size_t alignment, MemFlags flags) nothrow @nogc
         _pools[allocated_in].used += block;
         if (_pools[allocated_in].used > _pools[allocated_in].peak_used)
             _pools[allocated_in].peak_used = _pools[allocated_in].used;
+        note_pool_usage(allocated_in, _pools[allocated_in].used);
     }
     else
         log_oom(size, alignment, flags);
@@ -426,6 +429,7 @@ void[] realloc_impl(void[] mem, size_t new_size, size_t alignment, MemFlags flag
     owner.used = owner.used - old_block + new_block;
     if (owner.used > owner.peak_used)
         owner.peak_used = owner.used;
+    note_pool_usage(owner - _pools.ptr, owner.used);
     return p[0 .. new_size];
 }
 
@@ -436,6 +440,7 @@ void free_impl(void* ptr) nothrow @nogc
         return;
     owner.used -= tlsf_block_size(ptr);
     tlsf_free(owner.tlsf, ptr);
+    note_pool_usage(owner - _pools.ptr, owner.used);
 }
 
 void init_pools() nothrow @nogc

src/urt/driver/esp32/alloc.d

@@ -10,6 +10,7 @@ enum has_memsize  = true;
 enum has_exec     = true;
 enum has_retain   = true;
 enum has_memflags = true;
+enum has_pool_usage = true;
 
 void[] _alloc(size_t size, size_t alignment, MemFlags flags) pure
 {
@@ -40,12 +41,16 @@ void[] _alloc(size_t size, size_t alignment, MemFlags flags) pure
             (cast(LogFn) &log_alloc_oom)(size, alignment, flags);
         }
     }
-    return p ? p[0 .. size] : null;
+    if (p is null)
+        return null;
+    note_pools();
+    return p[0 .. size];
 }
 
 void _free(void* ptr) pure
 {
     heap_caps_aligned_free(ptr);
+    note_pools();
 }
 
 size_t _memsize(void* ptr) pure
@@ -78,6 +83,7 @@ void _free_retain(void[] mem) pure
 
 private:
 
+enum CAP_8BIT      = 1 << 2;
 enum CAP_DMA       = 1 << 3;
 enum CAP_SPIRAM    = 1 << 10;
 enum CAP_INTERNAL  = 1 << 11;
@@ -86,9 +92,49 @@ enum CAP_IRAM_8BIT = 1 << 13;
 enum CAP_RTCRAM    = 1 << 15;
 
 version (Iram8BitSlowMemory)
+{
     enum slow_caps = CAP_IRAM_8BIT;
+    enum slow_query_caps = CAP_IRAM_8BIT;
+}
 else
+{
     enum slow_caps = CAP_DEFAULT | CAP_SPIRAM;
+    enum slow_query_caps = CAP_SPIRAM;
+}
+
+// The pools urt.system reports, queried by the same caps so the watermarks and sysinfo
+// describe the same two heaps.
+immutable uint[2] _pool_caps = [CAP_INTERNAL | CAP_8BIT, slow_query_caps];
+__gshared size_t[2] _pool_total;
+__gshared bool _totals_valid;
+
+// Interval watermarks have to come off the IDF heap, not off a counter we keep: WiFi, lwIP and
+// the rest of IDF allocate without passing through here, and they are exactly the pressure worth
+// watching. heap_caps_get_free_size sums a per-heap counter over the registered region list, so
+// it is cheap enough per alloc; the totals are fixed once the regions register, and caching them
+// keeps a chip with no PSRAM from walking that list for an empty pool every time.
+void note_pools() pure
+{
+    static void impl() nothrow @nogc
+    {
+        import urt.mem.pressure : note_pool_usage;
+
+        if (!_totals_valid)
+        {
+            foreach (i, caps; _pool_caps)
+                _pool_total[i] = heap_caps_get_total_size(caps);
+            _totals_valid = true;
+        }
+        foreach (i, caps; _pool_caps)
+        {
+            if (_pool_total[i])
+                note_pool_usage(i, _pool_total[i] - heap_caps_get_free_size(caps));
+        }
+    }
+
+    alias Fn = void function() pure nothrow @nogc;
+    (cast(Fn) &impl)();
+}
 
 // MemFlags [2:0] -> ESP-IDF heap_caps
 //   [1:0] speed: 0=default, 1=fast, 2=slow, 3=fastest
@@ -107,6 +153,7 @@ immutable uint[8] _esp_caps = [
 extern(C) void* heap_caps_aligned_alloc(size_t alignment, size_t size, uint caps) pure;
 extern(C) void heap_caps_aligned_free(void* ptr) pure;
 extern(C) size_t heap_caps_get_allocated_size(void* ptr) pure;
+extern(C) size_t heap_caps_get_total_size(uint caps) pure;
 extern(C) size_t heap_caps_get_free_size(uint caps) pure;
 extern(C) size_t heap_caps_get_largest_free_block(uint caps) pure;
 

src/urt/driver/posix/alloc.d

@@ -10,6 +10,7 @@ enum has_memsize  = true;
 enum has_exec     = true;
 enum has_retain   = false;
 enum has_memflags = false;
+enum has_pool_usage = false;
 
 void[] _alloc(size_t size, size_t alignment, MemFlags) pure
 {

src/urt/driver/rp2350/alloc.d

@@ -10,6 +10,7 @@ enum has_memsize  = false;
 enum has_exec     = false;
 enum has_retain   = false;
 enum has_memflags = false;
+enum has_pool_usage = false;
 
 void[] _alloc(size_t size, size_t alignment, MemFlags) pure
 {

src/urt/driver/stm32/alloc.d

@@ -10,6 +10,7 @@ enum has_memsize  = false;
 enum has_exec     = false;
 enum has_retain   = false; // TODO: backup SRAM
 enum has_memflags = false; // TODO: TCM vs SRAM
+enum has_pool_usage = false;
 
 void[] _alloc(size_t size, size_t alignment, MemFlags) pure
 {

src/urt/driver/windows/alloc.d

@@ -13,6 +13,7 @@ enum has_memsize  = true;
 enum has_exec     = true;
 enum has_retain   = false;
 enum has_memflags = false;
+enum has_pool_usage = false;
 
 void[] _alloc(size_t size, size_t alignment, MemFlags) pure
 {

src/urt/mem/alloc.d

@@ -31,6 +31,8 @@ void[] alloc(size_t size, size_t alignment, MemFlags flags = MemFlags.none) pure
     assert(is_power_of_2(alignment), "Alignment must be a power of two!");
 
     void[] mem = _alloc(size, alignment, flags);
+    if (mem.ptr !is null)
+        account(mem.length, false);
     version (AllocTracking)
     {
         import urt.mem.profile.record : track_alloc;
@@ -69,6 +71,9 @@ void[] realloc(void[] mem, size_t new_size, size_t alignment = 8, MemFlags flags
         void* old_ptr = mem.ptr;
         size_t old_size = mem.length;
         void[] new_mem = _realloc(mem, new_size, alignment, flags);
+        if (new_mem.ptr !is null && new_mem.length != old_size)
+            account(new_mem.length > old_size ? new_mem.length - old_size : old_size - new_mem.length,
+                    new_mem.length < old_size);
         version (AllocTracking)
         {
             import urt.mem.profile.record : track_realloc;
@@ -107,6 +112,7 @@ void free(void[] mem) pure
 {
     if (mem.ptr is null)
         return;
+    account(mem.length, true);
     version (AllocTracking)
     {
         import urt.mem.profile.record : untrack_alloc;
@@ -139,6 +145,9 @@ void[] expand(void[] mem, size_t new_size) pure
         void[] new_mem = null;
         assert(false, "unsupported");
     }
+    if (new_mem.ptr !is null && new_mem.length != mem.length)
+        account(new_mem.length > mem.length ? new_mem.length - mem.length : mem.length - new_mem.length,
+                new_mem.length < mem.length);
     version (AllocProfile)
     {
         import urt.mem.profile.log : profile_expand;
@@ -196,6 +205,30 @@ void free_retain(void[] mem) pure
 }
 
 
+// Feed the interval watermarks for drivers that cannot name the pool a block came from. The
+// entry points above are `pure` and the accounting is not, so the crossing casts the impurity
+// away; collected here so the cast appears once. Drivers that do track their own per-pool usage
+// nudge the watermarks at the point they update it, and compile this out entirely.
+private void account(size_t bytes, bool freed) pure
+{
+    static if (!has_pool_usage)
+    {
+        static void impl(size_t bytes, bool freed) nothrow @nogc
+        {
+            import urt.mem.pressure : account_pool_alloc, account_pool_free;
+
+            if (freed)
+                account_pool_free(bytes);
+            else
+                account_pool_alloc(bytes);
+        }
+
+        alias Fn = void function(size_t, bool) pure nothrow @nogc;
+        (cast(Fn) &impl)(bytes, freed);
+    }
+}
+
+
 // pointer tagging utilities -- for containers to store flags in low 3 bits
 // of 8-byte aligned pointers. the allocator itself returns clean pointers.
 T* tag(T)(T* ptr, MemFlags flags) pure

src/urt/mem/pressure.d

@@ -0,0 +1,88 @@
+module urt.mem.pressure;
+
+import urt.atomic;
+
+nothrow @nogc:
+
+
+// Per-pool interval watermarks. Every alloc and free nudges its pool's pair, and a sampler reads
+// the pair and re-arms both to the pool's latest usage. An interval therefore reports the
+// extremes reached between two samples rather than the level at the sample instant, which is
+// what makes a transient spike or a creeping floor visible to a reader sampling once a second.
+// One sampler per pool: a second reader steals the first's interval. Note and sample race only
+// against each other's precision, and a lost update costs one sample of resolution, which is not
+// worth a CAS loop on the allocation path.
+
+enum MaxUsagePools = 4;
+
+void note_pool_usage(size_t pool, size_t used)
+{
+    Watermark* w = &_watermarks[pool];
+    atomicStore(w.current, used);
+    if (used < atomicLoad(w.low))
+        atomicStore(w.low, used);
+    if (used > atomicLoad(w.high))
+        atomicStore(w.high, used);
+}
+
+void sample_pool_usage(size_t pool, out size_t low, out size_t high)
+{
+    Watermark* w = &_watermarks[pool];
+    size_t current = atomicLoad(w.current);
+    low = atomicLoad(w.low);
+    high = atomicLoad(w.high);
+    atomicStore(w.low, current);
+    atomicStore(w.high, current);
+    if (low > high)     // untouched since the last sample, or never tracked at all
+        low = high = current;
+}
+
+// Allocators that cannot say which pool a block came from feed the whole heap through here as
+// pool 0. The running total lives here because on those platforms nothing else is counting it.
+void account_pool_alloc(size_t bytes)
+{
+    note_pool_usage(0, atomicFetchAdd(_untracked_used, bytes) + bytes);
+}
+
+void account_pool_free(size_t bytes)
+{
+    note_pool_usage(0, atomicFetchSub(_untracked_used, bytes) - bytes);
+}
+
+
+private:
+
+struct Watermark
+{
+    shared size_t current;
+    shared size_t low = size_t.max;
+    shared size_t high;
+}
+
+__gshared Watermark[MaxUsagePools] _watermarks;
+shared size_t _untracked_used;
+
+
+unittest
+{
+    // pools 2 and 3 are above what any allocator tracks, so nothing else can drift them
+    size_t low, high;
+    note_pool_usage(3, 1000);
+    note_pool_usage(3, 5000);
+    note_pool_usage(3, 2000);
+    sample_pool_usage(3, low, high);
+    assert(low == 1000 && high == 5000);
+
+    // an interval with nothing in it collapses onto the last level rather than reporting the
+    // previous window again
+    sample_pool_usage(3, low, high);
+    assert(low == 2000 && high == 2000);
+
+    note_pool_usage(3, 2500);
+    sample_pool_usage(3, low, high);
+    assert(low == 2000 && high == 2500);
+
+    // a pool the allocator never notes reads as zero, not as the arming sentinel
+    sample_pool_usage(2, low, high);
+    assert(low == 0 && high == 0);
+}