diff --git a/src/urt/driver/bk7231/alloc.d b/src/urt/driver/bk7231/alloc.d index af693c45..9008c60e 100644 --- a/src/urt/driver/bk7231/alloc.d +++ b/src/urt/driver/bk7231/alloc.d @@ -10,6 +10,7 @@ enum has_memsize = false; enum has_exec = false; enum has_retain = false; enum has_memflags = false; +enum has_pool_usage = false; void[] _alloc(size_t size, size_t alignment, MemFlags) pure { diff --git a/src/urt/driver/bl_common/alloc.d b/src/urt/driver/bl_common/alloc.d index 4b3957b4..2e7e7b2d 100644 --- a/src/urt/driver/bl_common/alloc.d +++ b/src/urt/driver/bl_common/alloc.d @@ -49,6 +49,7 @@ version (BouffaloUnifiedAlloc): import urt.attribute : fast_data; import urt.mem.alloc : MemFlags; +import urt.mem.pressure : note_pool_usage; @nogc nothrow: @@ -59,6 +60,7 @@ enum has_memsize = true; enum has_exec = false; enum has_retain = false; enum has_memflags = true; +enum has_pool_usage = true; void[] _alloc(size_t size, size_t alignment, MemFlags flags) pure @@ -399,6 +401,7 @@ void[] alloc_impl(size_t size, size_t alignment, MemFlags flags) nothrow @nogc _pools[allocated_in].used += block; if (_pools[allocated_in].used > _pools[allocated_in].peak_used) _pools[allocated_in].peak_used = _pools[allocated_in].used; + note_pool_usage(allocated_in, _pools[allocated_in].used); } else log_oom(size, alignment, flags); @@ -426,6 +429,7 @@ void[] realloc_impl(void[] mem, size_t new_size, size_t alignment, MemFlags flag owner.used = owner.used - old_block + new_block; if (owner.used > owner.peak_used) owner.peak_used = owner.used; + note_pool_usage(owner - _pools.ptr, owner.used); return p[0 .. new_size]; } @@ -436,6 +440,7 @@ void free_impl(void* ptr) nothrow @nogc return; owner.used -= tlsf_block_size(ptr); tlsf_free(owner.tlsf, ptr); + note_pool_usage(owner - _pools.ptr, owner.used); } void init_pools() nothrow @nogc diff --git a/src/urt/driver/esp32/alloc.d b/src/urt/driver/esp32/alloc.d index 62629a1a..745f2d3d 100644 --- a/src/urt/driver/esp32/alloc.d +++ b/src/urt/driver/esp32/alloc.d @@ -10,6 +10,7 @@ enum has_memsize = true; enum has_exec = true; enum has_retain = true; enum has_memflags = true; +enum has_pool_usage = true; void[] _alloc(size_t size, size_t alignment, MemFlags flags) pure { @@ -40,12 +41,16 @@ void[] _alloc(size_t size, size_t alignment, MemFlags flags) pure (cast(LogFn) &log_alloc_oom)(size, alignment, flags); } } - return p ? p[0 .. size] : null; + if (p is null) + return null; + note_pools(); + return p[0 .. size]; } void _free(void* ptr) pure { heap_caps_aligned_free(ptr); + note_pools(); } size_t _memsize(void* ptr) pure @@ -78,6 +83,7 @@ void _free_retain(void[] mem) pure private: +enum CAP_8BIT = 1 << 2; enum CAP_DMA = 1 << 3; enum CAP_SPIRAM = 1 << 10; enum CAP_INTERNAL = 1 << 11; @@ -86,9 +92,49 @@ enum CAP_IRAM_8BIT = 1 << 13; enum CAP_RTCRAM = 1 << 15; version (Iram8BitSlowMemory) +{ enum slow_caps = CAP_IRAM_8BIT; + enum slow_query_caps = CAP_IRAM_8BIT; +} else +{ enum slow_caps = CAP_DEFAULT | CAP_SPIRAM; + enum slow_query_caps = CAP_SPIRAM; +} + +// The pools urt.system reports, queried by the same caps so the watermarks and sysinfo +// describe the same two heaps. +immutable uint[2] _pool_caps = [CAP_INTERNAL | CAP_8BIT, slow_query_caps]; +__gshared size_t[2] _pool_total; +__gshared bool _totals_valid; + +// Interval watermarks have to come off the IDF heap, not off a counter we keep: WiFi, lwIP and +// the rest of IDF allocate without passing through here, and they are exactly the pressure worth +// watching. heap_caps_get_free_size sums a per-heap counter over the registered region list, so +// it is cheap enough per alloc; the totals are fixed once the regions register, and caching them +// keeps a chip with no PSRAM from walking that list for an empty pool every time. +void note_pools() pure +{ + static void impl() nothrow @nogc + { + import urt.mem.pressure : note_pool_usage; + + if (!_totals_valid) + { + foreach (i, caps; _pool_caps) + _pool_total[i] = heap_caps_get_total_size(caps); + _totals_valid = true; + } + foreach (i, caps; _pool_caps) + { + if (_pool_total[i]) + note_pool_usage(i, _pool_total[i] - heap_caps_get_free_size(caps)); + } + } + + alias Fn = void function() pure nothrow @nogc; + (cast(Fn) &impl)(); +} // MemFlags [2:0] -> ESP-IDF heap_caps // [1:0] speed: 0=default, 1=fast, 2=slow, 3=fastest @@ -107,6 +153,7 @@ immutable uint[8] _esp_caps = [ extern(C) void* heap_caps_aligned_alloc(size_t alignment, size_t size, uint caps) pure; extern(C) void heap_caps_aligned_free(void* ptr) pure; extern(C) size_t heap_caps_get_allocated_size(void* ptr) pure; +extern(C) size_t heap_caps_get_total_size(uint caps) pure; extern(C) size_t heap_caps_get_free_size(uint caps) pure; extern(C) size_t heap_caps_get_largest_free_block(uint caps) pure; diff --git a/src/urt/driver/posix/alloc.d b/src/urt/driver/posix/alloc.d index 2f9de46b..7d4c13e9 100644 --- a/src/urt/driver/posix/alloc.d +++ b/src/urt/driver/posix/alloc.d @@ -10,6 +10,7 @@ enum has_memsize = true; enum has_exec = true; enum has_retain = false; enum has_memflags = false; +enum has_pool_usage = false; void[] _alloc(size_t size, size_t alignment, MemFlags) pure { diff --git a/src/urt/driver/rp2350/alloc.d b/src/urt/driver/rp2350/alloc.d index 44d71e9c..d264e7db 100644 --- a/src/urt/driver/rp2350/alloc.d +++ b/src/urt/driver/rp2350/alloc.d @@ -10,6 +10,7 @@ enum has_memsize = false; enum has_exec = false; enum has_retain = false; enum has_memflags = false; +enum has_pool_usage = false; void[] _alloc(size_t size, size_t alignment, MemFlags) pure { diff --git a/src/urt/driver/stm32/alloc.d b/src/urt/driver/stm32/alloc.d index 806bbd5f..837c4259 100644 --- a/src/urt/driver/stm32/alloc.d +++ b/src/urt/driver/stm32/alloc.d @@ -10,6 +10,7 @@ enum has_memsize = false; enum has_exec = false; enum has_retain = false; // TODO: backup SRAM enum has_memflags = false; // TODO: TCM vs SRAM +enum has_pool_usage = false; void[] _alloc(size_t size, size_t alignment, MemFlags) pure { diff --git a/src/urt/driver/windows/alloc.d b/src/urt/driver/windows/alloc.d index 4ebf83b9..7956500e 100644 --- a/src/urt/driver/windows/alloc.d +++ b/src/urt/driver/windows/alloc.d @@ -13,6 +13,7 @@ enum has_memsize = true; enum has_exec = true; enum has_retain = false; enum has_memflags = false; +enum has_pool_usage = false; void[] _alloc(size_t size, size_t alignment, MemFlags) pure { diff --git a/src/urt/mem/alloc.d b/src/urt/mem/alloc.d index 84c97eae..9a3d194b 100644 --- a/src/urt/mem/alloc.d +++ b/src/urt/mem/alloc.d @@ -31,6 +31,8 @@ void[] alloc(size_t size, size_t alignment, MemFlags flags = MemFlags.none) pure assert(is_power_of_2(alignment), "Alignment must be a power of two!"); void[] mem = _alloc(size, alignment, flags); + if (mem.ptr !is null) + account(mem.length, false); version (AllocTracking) { import urt.mem.profile.record : track_alloc; @@ -69,6 +71,9 @@ void[] realloc(void[] mem, size_t new_size, size_t alignment = 8, MemFlags flags void* old_ptr = mem.ptr; size_t old_size = mem.length; void[] new_mem = _realloc(mem, new_size, alignment, flags); + if (new_mem.ptr !is null && new_mem.length != old_size) + account(new_mem.length > old_size ? new_mem.length - old_size : old_size - new_mem.length, + new_mem.length < old_size); version (AllocTracking) { import urt.mem.profile.record : track_realloc; @@ -107,6 +112,7 @@ void free(void[] mem) pure { if (mem.ptr is null) return; + account(mem.length, true); version (AllocTracking) { import urt.mem.profile.record : untrack_alloc; @@ -139,6 +145,9 @@ void[] expand(void[] mem, size_t new_size) pure void[] new_mem = null; assert(false, "unsupported"); } + if (new_mem.ptr !is null && new_mem.length != mem.length) + account(new_mem.length > mem.length ? new_mem.length - mem.length : mem.length - new_mem.length, + new_mem.length < mem.length); version (AllocProfile) { import urt.mem.profile.log : profile_expand; @@ -196,6 +205,30 @@ void free_retain(void[] mem) pure } +// Feed the interval watermarks for drivers that cannot name the pool a block came from. The +// entry points above are `pure` and the accounting is not, so the crossing casts the impurity +// away; collected here so the cast appears once. Drivers that do track their own per-pool usage +// nudge the watermarks at the point they update it, and compile this out entirely. +private void account(size_t bytes, bool freed) pure +{ + static if (!has_pool_usage) + { + static void impl(size_t bytes, bool freed) nothrow @nogc + { + import urt.mem.pressure : account_pool_alloc, account_pool_free; + + if (freed) + account_pool_free(bytes); + else + account_pool_alloc(bytes); + } + + alias Fn = void function(size_t, bool) pure nothrow @nogc; + (cast(Fn) &impl)(bytes, freed); + } +} + + // pointer tagging utilities -- for containers to store flags in low 3 bits // of 8-byte aligned pointers. the allocator itself returns clean pointers. T* tag(T)(T* ptr, MemFlags flags) pure diff --git a/src/urt/mem/pressure.d b/src/urt/mem/pressure.d new file mode 100644 index 00000000..ee6d6716 --- /dev/null +++ b/src/urt/mem/pressure.d @@ -0,0 +1,88 @@ +module urt.mem.pressure; + +import urt.atomic; + +nothrow @nogc: + + +// Per-pool interval watermarks. Every alloc and free nudges its pool's pair, and a sampler reads +// the pair and re-arms both to the pool's latest usage. An interval therefore reports the +// extremes reached between two samples rather than the level at the sample instant, which is +// what makes a transient spike or a creeping floor visible to a reader sampling once a second. +// One sampler per pool: a second reader steals the first's interval. Note and sample race only +// against each other's precision, and a lost update costs one sample of resolution, which is not +// worth a CAS loop on the allocation path. + +enum MaxUsagePools = 4; + +void note_pool_usage(size_t pool, size_t used) +{ + Watermark* w = &_watermarks[pool]; + atomicStore(w.current, used); + if (used < atomicLoad(w.low)) + atomicStore(w.low, used); + if (used > atomicLoad(w.high)) + atomicStore(w.high, used); +} + +void sample_pool_usage(size_t pool, out size_t low, out size_t high) +{ + Watermark* w = &_watermarks[pool]; + size_t current = atomicLoad(w.current); + low = atomicLoad(w.low); + high = atomicLoad(w.high); + atomicStore(w.low, current); + atomicStore(w.high, current); + if (low > high) // untouched since the last sample, or never tracked at all + low = high = current; +} + +// Allocators that cannot say which pool a block came from feed the whole heap through here as +// pool 0. The running total lives here because on those platforms nothing else is counting it. +void account_pool_alloc(size_t bytes) +{ + note_pool_usage(0, atomicFetchAdd(_untracked_used, bytes) + bytes); +} + +void account_pool_free(size_t bytes) +{ + note_pool_usage(0, atomicFetchSub(_untracked_used, bytes) - bytes); +} + + +private: + +struct Watermark +{ + shared size_t current; + shared size_t low = size_t.max; + shared size_t high; +} + +__gshared Watermark[MaxUsagePools] _watermarks; +shared size_t _untracked_used; + + +unittest +{ + // pools 2 and 3 are above what any allocator tracks, so nothing else can drift them + size_t low, high; + note_pool_usage(3, 1000); + note_pool_usage(3, 5000); + note_pool_usage(3, 2000); + sample_pool_usage(3, low, high); + assert(low == 1000 && high == 5000); + + // an interval with nothing in it collapses onto the last level rather than reporting the + // previous window again + sample_pool_usage(3, low, high); + assert(low == 2000 && high == 2000); + + note_pool_usage(3, 2500); + sample_pool_usage(3, low, high); + assert(low == 2000 && high == 2500); + + // a pool the allocator never notes reads as zero, not as the arming sentinel + sample_pool_usage(2, low, high); + assert(low == 0 && high == 0); +} diff --git a/src/urt/system.d b/src/urt/system.d index 964b5bcf..422ef66c 100644 --- a/src/urt/system.d +++ b/src/urt/system.d @@ -1,5 +1,6 @@ module urt.system; +import urt.mem.pressure : MaxUsagePools, sample_pool_usage; import urt.platform; import urt.processor; import urt.time; @@ -92,9 +93,12 @@ struct MemoryPool ulong used; // currently allocated ulong peak_used; // high-water mark of used (0 if unavailable) ulong largest_free; // largest contiguous allocatable block (0 if unknown) + ulong low; // interval watermarks; only sample_memory_watermarks() fills these + ulong high; } enum MaxMemoryPools = 4; +static assert(MaxMemoryPools <= MaxUsagePools, "the allocator tracks fewer pools than sysinfo reports"); struct SystemInfo { @@ -203,6 +207,22 @@ SystemInfo get_sysinfo() return r; } +// The least and greatest usage the allocator saw between this call and the previous one, which +// is where a transient spike or a creeping floor shows itself -- a once-a-second reading of +// `used` walks straight past both. Reading re-arms the interval, so exactly one caller owns the +// cadence. Where the allocator owns the pool outright (the embedded targets) the watermarks +// follow `used`; elsewhere they follow urt's own heap total, which `used` does not report. +void sample_memory_watermarks(ref SystemInfo info) +{ + foreach (i, ref p; info.pools) + { + size_t low, high; + sample_pool_usage(i, low, high); + p.low = low; + p.high = high; + } +} + void set_system_idle_params(IdleParams params) { version (Windows) @@ -227,33 +247,11 @@ void set_system_idle_params(IdleParams params) static assert(0, "Not implemented"); } +// `reference` is when the loop went idle and the wake is now, so everything between the two +// is idle and everything since the previous wake was busy. void count_system_load(MonoTime reference) { - MonoTime now = get_time(); - - size_t a = cast(size_t)(reference - MonoTime()).as!"nsecs"; - size_t b = cast(size_t)(now - MonoTime()).as!"nsecs"; - - import urt.util : log2; - enum shift = log2(cpu_bucket_len); - size_t full_intervals = (b >> shift) - (a >> shift); - - if (full_intervals == 0) - g_cpu_time[g_bucket] += b - a; - else - { - enum mask = cpu_counter_buckets - 1; - enum cpu_bucket_mask = cpu_bucket_len - 1; - - if (full_intervals > cpu_counter_buckets) - full_intervals = cpu_counter_buckets; - - g_cpu_time[g_bucket++] += cpu_bucket_len - (a & cpu_bucket_mask); - for (uint i = 1; i < full_intervals; i++) - g_cpu_time[g_bucket++ & mask] = cpu_bucket_len; - g_bucket = g_bucket & mask; - g_cpu_time[g_bucket] = b & cpu_bucket_mask; - } + account_idle((reference - MonoTime()).as!"usecs", (get_time() - MonoTime()).as!"usecs"); } uint get_cpu_load() @@ -264,7 +262,24 @@ uint get_cpu_load() idle_time += g_cpu_time[i]; enum total_time = cpu_bucket_len*(cpu_counter_buckets - 1); uint cpu_time = total_time - idle_time; - return cast(uint)(cpu_time*100 / total_time); + return cpu_time * 100 / total_time; +} + +// get_cpu_load() averages the whole ring. This reports the quietest and busiest single bucket +// in it, so a burst that saturates one ~67ms slice shows through instead of being averaged flat. +void get_cpu_load_range(out uint low, out uint high) +{ + low = 100; + for (uint i = 0; i < cpu_counter_buckets; i++) + { + if (i == g_bucket) // still filling, so not yet a whole slice + continue; + uint load = (cpu_bucket_len - g_cpu_time[i]) * 100 / cpu_bucket_len; + if (load < low) + low = load; + if (load > high) + high = load; + } } unittest @@ -281,6 +296,58 @@ unittest writelnf(" {0}: {1}kb used / {2}kb total (peak {3}kb)", p.name, p.used / 1024, p.total / 1024, p.peak_used / 1024); } + + // cpu load accounting, driven through account_idle so the timestamps are ours to choose + static void reset_load_ring() + { + g_cpu_time[] = 0; + g_bucket = 0; + g_bucket_base = 0; + } + + // a 20Hz loop busy for a quarter of every period. The busy stretch crosses bucket + // boundaries of its own, which is what used to push buckets past their capacity. + reset_load_ring(); + foreach (i; 0 .. 400) + account_idle(i * 50_000 + 12_500, i * 50_000 + 50_000); + foreach (c; g_cpu_time) + assert(c <= cpu_bucket_len, "a bucket cannot hold more idle than it is long"); + uint load = get_cpu_load(); + assert(load >= 23 && load <= 27, "a quarter busy should read as roughly 25%"); + + // fully busy and fully idle are the ends of the range + reset_load_ring(); + foreach (i; 0 .. 400) + account_idle(i * 50_000 + 50_000, i * 50_000 + 50_000); + assert(get_cpu_load() == 100); + reset_load_ring(); + foreach (i; 0 .. 400) + account_idle(i * 50_000, i * 50_000 + 50_000); + assert(get_cpu_load() == 0); + + // the timestamps outrun a 32-bit microsecond counter after ~72 minutes, so the accounting + // has to keep working either side of that + reset_load_ring(); + ulong base = (1UL << 32) - 5_000_000; + foreach (i; 0 .. 400) + account_idle(base + i * 50_000 + 12_500, base + i * 50_000 + 50_000); + load = get_cpu_load(); + assert(load >= 23 && load <= 27, "the bucket number must not wrap with the 32-bit stamp"); + + // one saturated bucket inside an otherwise quiet second: the average hides it, the range + // is the whole reason this pair exists + reset_load_ring(); + foreach (i; 0 .. 100) + { + bool burst = i >= 96; + account_idle(i * 50_000 + (burst ? 50_000 : 2_500), i * 50_000 + 50_000); + } + uint low, high; + get_cpu_load_range(low, high); + assert(get_cpu_load() < 30 && high > 70, "a saturated bucket must show in the range"); + assert(low < 20, "the quiet buckets must still read quiet"); + + reset_load_ring(); } @@ -288,11 +355,60 @@ package: import urt.attribute : fast_data; -enum uint cpu_bucket_len = 0x400_0000; // nanosecond buckets of ~67ms +// Microsecond buckets of ~65ms. Nanoseconds put total_time near 1e9, so the percentage in +// get_cpu_load needed 64-bit arithmetic to survive its own multiply; microseconds keep a whole +// window inside 20 bits and leave the load calculation as 32-bit multiply and divide, which is +// what the 32-bit targets want. +enum uint cpu_bucket_len = 0x1_0000; enum cpu_counter_buckets = 16; __gshared @fast_data uint[16] g_cpu_time; __gshared @fast_data ubyte g_bucket = 0; +// absolute bucket number sitting in g_bucket; 64 bits so it never wraps, and only ever +// shifted and compared, so a 32-bit target pays nothing for the width +__gshared @fast_data ulong g_bucket_base; + +// Both stamps are microseconds since the monotonic epoch. Roll to where the idle span begins +// before crediting anything: the busy stretch since the previous wake can itself cross a +// boundary, and the buckets it crossed hold no idle at all. Measuring the span from `idle_from` +// alone leaves the ring pointing wherever it was and dumps this idle into a bucket that closed +// a while ago, taking it over capacity. +void account_idle(ulong idle_from, ulong idle_to) +{ + import urt.util : log2; + enum shift = log2(cpu_bucket_len); + enum uint offset_mask = cpu_bucket_len - 1; + + roll_cpu_buckets(idle_from >> shift, 0); + + uint from_offset = idle_from & offset_mask; + uint to_offset = idle_to & offset_mask; + if ((idle_to >> shift) == g_bucket_base) + { + g_cpu_time[g_bucket] += to_offset - from_offset; + return; + } + + g_cpu_time[g_bucket] += cpu_bucket_len - from_offset; + roll_cpu_buckets(idle_to >> shift, cpu_bucket_len); // buckets lying wholly inside the span + g_cpu_time[g_bucket] = to_offset; +} + +// step the ring forward to absolute bucket `to`, writing `fill` into every bucket passed over +void roll_cpu_buckets(ulong to, uint fill) +{ + ulong steps = to - g_bucket_base; + if (steps == 0) + return; + if (steps > cpu_counter_buckets) + steps = cpu_counter_buckets; + foreach (i; 0 .. steps) + { + g_bucket = (g_bucket + 1) & (cpu_counter_buckets - 1); + g_cpu_time[g_bucket] = fill; + } + g_bucket_base = to; +} version (Bouffalo) {