From 8a751335e1fd06f052b5137e000e1318203c2548 Mon Sep 17 00:00:00 2001 From: Dennis Paler Date: Mon, 3 Aug 2026 11:11:58 +0800 Subject: [PATCH 1/3] docs: update readme --- README.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/README.md b/README.md index 9e57a3d..a94f751 100644 --- a/README.md +++ b/README.md @@ -153,11 +153,6 @@ See [recached.dev/roadmap](https://recached.dev/roadmap) for what's planned. Reach out: [dennis@thinkgrid.dev](mailto:dennis@thinkgrid.dev) - -## Support Recached - -Recached is free and open-source, maintained by one person. If it saves you infrastructure cost or development time, [sponsoring on GitHub](https://github.com/sponsors/thinkgrid-labs) directly funds continued development: more Redis commands, RESP3, cluster support, and performance work. - ## License Apache License 2.0 — © 2026 ThinkGrid Labs From 8d3ce42d9fc30cf4d9b4f46260a37d71d30abde8 Mon Sep 17 00:00:00 2001 From: Dennis Paler Date: Mon, 3 Aug 2026 13:20:41 +0800 Subject: [PATCH 2/3] feat(server): MEMORY USAGE, PUBSUB introspection, MODULE LIST, and INFO's # Cluster section --- CHANGELOG.md | 64 +++++ core-engine/src/catalog.rs | 46 ++++ core-engine/src/cmd.rs | 134 ++++++++++ core-engine/src/store.rs | 98 ++++++++ docs/server/commands.md | 58 ++++- server-native/src/main.rs | 486 +++++++++++++++++++++++++++++++++++++ 6 files changed, 885 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fda474b..956e2ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,70 @@ All notable changes to Recached are documented here. --- +## [Unreleased] + +### Added + +- **`MEMORY USAGE key [SAMPLES count]`.** Recached enforces `maxmemory` and evicts against + it, but an operator who hit the limit had no command that answered "which key is eating + it?" — `INFO memory` gives one total and nothing per key. The reply is the same figure the + eviction loop bills the key for, computed by the same function, so "what is this costing me" + and "what gets evicted next" cannot drift apart the way two separate estimators would. + A missing or expired key reports nil, not zero: "no such key" and "an empty key" are + different facts. `SAMPLES` is parsed and its count discarded — Redis uses it to bound how + much of a nested value it walks before extrapolating, and Recached always walks all of it, + so the reply is never less accurate than what was asked for. `MEMORY DOCTOR`, `STATS`, + `PURGE` and `MALLOC-STATS` are refused with a reason: they describe an allocator arena that + Recached does not manage and cannot honestly report on. + +- **`PUBSUB CHANNELS [pattern]`, `PUBSUB NUMSUB [channel ...]`, `PUBSUB NUMPAT`.** Pub/sub has + shipped since the first release with no way to see any of it — `PUBLISH` returned a delivery + count and that was the only observable, so "is anything actually subscribed?" could only be + answered by publishing and watching the number. The subscriber hub already held both + registries; these are a read of state that existed all along. `CHANNELS` lists only channels + with a live subscriber and drops one the moment its last subscriber leaves, `NUMSUB` reports + a zero for a channel nobody is on rather than omitting it so the reply can be read by + position, and `NUMPAT` counts distinct patterns, not subscribers. Verified reply-for-reply + against redis-server 7.2.5. `PUBSUB` is classified as an admin command, so it is refused on + scope-limited WebSocket connections: a scoped connection may still `SUBSCRIBE` to any channel + it can name, but naming one and enumerating everyone else's are different powers, the same + line `GET` and `KEYS` sit on. + +- **`MODULE LIST`.** An empty array, which is the answer a stock `redis-server` with no modules + gives, and which lets a tool tell "no modules" apart from "cannot ask". `MODULE LOAD`, + `LOADEX` and `UNLOAD` are refused rather than answered `+OK`. + +- **`INFO` now reports the `# Cluster` section** — `cluster_enabled:0`, in the default set, so a + bare `INFO` carries it. This is how a cluster-aware client actually learns it is talking to a + single node, and Recached had no way to say so. + +### Changed + +- **`CLUSTER` is now refused with Redis's own sentence** — `ERR This instance has cluster support + disabled` — instead of `ERR unknown command`. Measuring beat assuming here: a `redis-server` + not started in cluster mode does **not** answer `CLUSTER INFO` with `cluster_enabled:0`, it + rejects the whole container with that error and publishes the flag through `INFO`. So the fix + was not to implement `CLUSTER INFO` — implementing it would have made Recached *less* like + Redis — but to copy the refusal and add the `INFO` section. `unknown command` was the one + reply a client cannot act on, because it reads as "too old to ask" rather than "not a cluster". + +- **`PUBSUB SHARDCHANNELS` and `SHARDNUMSUB` are refused**, where a standalone Redis answers both + with an empty array. The one deliberate divergence in this batch: Redis's empty array sits next + to a working `SSUBSCRIBE`, and Recached implements neither `SSUBSCRIBE` nor `SPUBLISH`, so the + same reply would invite a follow-up call that fails. + +### Fixed + +- **The command reference no longer claims Recached exports latency histograms.** `INFO`'s "not + implemented" note said per-command latency was on the Prometheus endpoint instead — it is not, + and never was. Recached has no latency instrumentation at all: `recached_commands_total` counts + calls and `recached_command_errors_total` counts failures, and neither says how long anything + took. The docs now say so plainly and point at the bounded reads (`HSCAN`, `SSCAN`, `ZSCAN`, + `GETRANGE`) as the way to avoid the slow paths, since there is no way to catch them after the + fact. `SLOWLOG` and latency histograms remain unimplemented. + +--- + ## [0.2.4] — 2026-08-02 ### Added diff --git a/core-engine/src/catalog.rs b/core-engine/src/catalog.rs index 9582d35..6d8f670 100644 --- a/core-engine/src/catalog.rs +++ b/core-engine/src/catalog.rs @@ -1165,6 +1165,52 @@ pub const CATALOG: &[CommandSpec] = &[ "core", "Reports the command catalog: COUNT, LIST, INFO and DOCS.", ), + // Container rows carry the key positions Redis reports for the container + // itself, which are 0/0/0 even for `MEMORY`, whose `USAGE` subcommand does + // take a key at position 2. Redis publishes that through the subcommand + // tree, which Recached does not implement; copying the container row keeps + // a proxy reading the same numbers here as from the server it was built + // against, and Recached has no shards to route to in any case. + spec( + "cluster", + -2, + &["stale"], + 0, + 0, + 0, + "core", + "Cluster topology. Recached is standalone: the container is refused, and INFO reports cluster_enabled:0.", + ), + spec( + "memory", + -2, + &["readonly"], + 0, + 0, + 0, + "keys", + "MEMORY USAGE reports the approximate bytes one key holds. The allocator subcommands are refused.", + ), + spec( + "module", + -2, + &["admin", "noscript"], + 0, + 0, + 0, + "core", + "Lists loaded modules. Recached has no module API, so the list is always empty.", + ), + spec( + "pubsub", + -2, + &["pubsub", "loading", "stale"], + 0, + 0, + 0, + "pub/sub", + "Introspects the pub/sub system: CHANNELS, NUMSUB and NUMPAT.", + ), // ── Recached-only ───────────────────────────────────────────────────── // No Redis counterpart, so these rows are declared rather than transcribed. spec( diff --git a/core-engine/src/cmd.rs b/core-engine/src/cmd.rs index b774bde..0427d00 100644 --- a/core-engine/src/cmd.rs +++ b/core-engine/src/cmd.rs @@ -110,6 +110,22 @@ pub enum Command { Config(Vec), /// `COMMAND [subcommand] [arg ...]` — the command catalog. CommandQuery(Vec), + /// `CLUSTER [arg ...]` — cluster topology. Recached is always + /// standalone, so the answer is fixed; it still has to be sayable, because + /// a client that cannot ask assumes the worst rather than assuming none. + Cluster(Vec), + /// `MODULE [arg ...]` — loaded modules. Recached has no module + /// API at all, which makes the list empty rather than unanswerable. + Module(Vec), + /// `PUBSUB [arg ...]` — pub/sub introspection. Connection-level + /// for the same reason as CLIENT: the subscriber hub lives in the server, + /// and the store has never heard of a channel. + PubSub(Vec), + /// `MEMORY [arg ...]` for every subcommand except `USAGE`, + /// which is [`Command::MemoryUsage`]. Split because the two are not the + /// same kind of command: `USAGE` reads one key and is scoped like any other + /// key read, while the rest describe the allocator and are not. + Memory(Vec), // ── Strings ────────────────────────────────────────────────────────────── Set(String, Vec, SetOptions), Get(String), @@ -151,6 +167,10 @@ pub enum Command { FlushDb, Rename(String, String), Type(String), + /// `MEMORY USAGE key [SAMPLES n]` — approximate bytes held by one key. + /// A key read, and scoped like one: it reports on a key's contents, so a + /// connection that may not read the key may not measure it either. + MemoryUsage(String), // ── Hash ───────────────────────────────────────────────────────────────── HSet(String, Vec<(String, Vec)>), HGet(String, String), @@ -362,6 +382,51 @@ impl Command { Ok(Command::Config(collect_strings(&mut arr[1..]))) } "COMMAND" => Ok(Command::CommandQuery(collect_strings(&mut arr[1..]))), + "CLUSTER" => { + need!(2); + Ok(Command::Cluster(collect_strings(&mut arr[1..]))) + } + "MODULE" => { + need!(2); + Ok(Command::Module(collect_strings(&mut arr[1..]))) + } + "PUBSUB" => { + need!(2); + Ok(Command::PubSub(collect_strings(&mut arr[1..]))) + } + "MEMORY" => { + need!(2); + let sub = extract_string(&arr[1]).unwrap_or_default().to_uppercase(); + if sub != "USAGE" { + return Ok(Command::Memory(collect_strings(&mut arr[1..]))); + } + need!(3); + let key = take_key(&mut arr[2])?; + // Redis accepts `SAMPLES n` to bound how much of a + // nested value it walks. Recached walks all of it — + // `entry_size` is the same measurement the eviction + // loop runs, and there is no cheaper approximation to + // fall back to — so the option is accepted and the + // count ignored. The reply is never less accurate than + // what was asked for, only more. + match arr.len() { + 3 => {} + 5 if extract_string(&arr[3]) + .unwrap_or_default() + .eq_ignore_ascii_case("SAMPLES") => + { + // Redis answers a negative SAMPLES with a plain + // syntax error rather than a range error; + // verified against 7.2.5 rather than assumed. + let n = extract_int(&arr[4])?; + if n < 0 { + return Err("ERR syntax error".to_string()); + } + } + _ => return Err("ERR syntax error".to_string()), + } + Ok(Command::MemoryUsage(key)) + } // ── Strings ─────────────────────────────────────────────── "SET" => { @@ -2062,6 +2127,75 @@ mod tests { assert!(Command::from_value(array(&["CONFIG"])).is_err()); } + #[test] + fn memory_usage_parses_as_a_key_read_and_the_rest_as_a_container() { + assert_eq!( + Command::from_value(array(&["MEMORY", "USAGE", "k"])).unwrap(), + Command::MemoryUsage("k".into()) + ); + // SAMPLES is accepted and its count discarded — the measurement always + // walks the whole value, so there is nothing for the count to bound. + assert_eq!( + Command::from_value(array(&["MEMORY", "usage", "k", "samples", "5"])).unwrap(), + Command::MemoryUsage("k".into()) + ); + assert_eq!( + Command::from_value(array(&["MEMORY", "USAGE", "k", "SAMPLES", "0"])).unwrap(), + Command::MemoryUsage("k".into()) + ); + // Every other subcommand stays a container command, answered by the + // server layer, so that MEMORY HELP and MEMORY DOCTOR get their own + // replies instead of a parse error naming the wrong thing. + assert_eq!( + Command::from_value(array(&["MEMORY", "DOCTOR"])).unwrap(), + Command::Memory(vec!["DOCTOR".into()]) + ); + assert!(Command::from_value(array(&["MEMORY"])).is_err()); + } + + #[test] + fn memory_usage_rejects_what_redis_rejects() { + // Each of these is `ERR syntax error` on redis-server 7.2.5, including + // the negative SAMPLES — which is a syntax error there, not a range + // error, so it is one here too. + for bad in [ + vec!["MEMORY", "USAGE"], + vec!["MEMORY", "USAGE", "k", "SAMPLES"], + vec!["MEMORY", "USAGE", "k", "SAMPLES", "-1"], + vec!["MEMORY", "USAGE", "k", "BOGUS", "1"], + vec!["MEMORY", "USAGE", "k", "SAMPLES", "1", "EXTRA"], + ] { + assert!( + Command::from_value(array(&bad)).is_err(), + "{bad:?} should not parse" + ); + } + assert_eq!( + Command::from_value(array(&["MEMORY", "USAGE", "k", "SAMPLES", "-1"])).unwrap_err(), + "ERR syntax error" + ); + } + + #[test] + fn cluster_module_and_pubsub_parse_as_containers() { + assert_eq!( + Command::from_value(array(&["CLUSTER", "INFO"])).unwrap(), + Command::Cluster(vec!["INFO".into()]) + ); + assert_eq!( + Command::from_value(array(&["MODULE", "LIST"])).unwrap(), + Command::Module(vec!["LIST".into()]) + ); + assert_eq!( + Command::from_value(array(&["PUBSUB", "NUMSUB", "a", "b"])).unwrap(), + Command::PubSub(vec!["NUMSUB".into(), "a".into(), "b".into()]) + ); + // A bare container is an arity error, as it is for CLIENT and CONFIG. + assert!(Command::from_value(array(&["CLUSTER"])).is_err()); + assert!(Command::from_value(array(&["MODULE"])).is_err()); + assert!(Command::from_value(array(&["PUBSUB"])).is_err()); + } + #[test] fn getrange_parse() { assert_eq!( diff --git a/core-engine/src/store.rs b/core-engine/src/store.rs index 67a77a2..64ebab3 100644 --- a/core-engine/src/store.rs +++ b/core-engine/src/store.rs @@ -1508,6 +1508,18 @@ impl KeyValueStore { Command::CommandQuery(_) => Value::Error( "ERR COMMAND is handled by the connection layer, not the store".to_string(), ), + Command::Cluster(_) => Value::Error( + "ERR CLUSTER is handled by the connection layer, not the store".to_string(), + ), + Command::Module(_) => Value::Error( + "ERR MODULE is handled by the connection layer, not the store".to_string(), + ), + Command::PubSub(_) => Value::Error( + "ERR PUBSUB is handled by the connection layer, not the store".to_string(), + ), + Command::Memory(_) => Value::Error( + "ERR MEMORY is handled by the connection layer, not the store".to_string(), + ), // ── Strings ─────────────────────────────────────────────────────── Command::Set(key, val, opts) => { @@ -1906,6 +1918,17 @@ impl KeyValueStore { _ => Value::SimpleString("none".to_string()), } } + Command::MemoryUsage(key) => { + let now = now_ms(); + match self.data.get(&key) { + // The same figure the eviction loop bills the key for, so + // "which key is eating my maxmemory" and "which key gets + // evicted next" are answered from one measurement rather + // than two that can disagree. + Some(e) if !e.is_expired(now) => Value::Integer(entry_size(&key, &e) as i64), + _ => Value::BulkString(None), + } + } // ── Hash ────────────────────────────────────────────────────────── Command::HSet(key, pairs) => { @@ -3364,6 +3387,11 @@ fn write_cost(cmd: &Command) -> usize { | Command::Dedup(_, _, _) | Command::QSub(_) | Command::QUnsub(_) + | Command::Cluster(_) + | Command::Module(_) + | Command::PubSub(_) + | Command::Memory(_) + | Command::MemoryUsage(_) | Command::Save | Command::BgSave | Command::LastSave @@ -6817,6 +6845,76 @@ mod metrics_tests { )); assert!(s.approximate_memory_bytes() >= empty + 4096); } + + #[test] + fn memory_usage_reports_a_key_and_nil_for_one_that_is_not_there() { + let s = KeyValueStore::new(); + // Redis answers a missing key with a nil bulk string, not zero: "no + // such key" and "an empty key" are different facts. + assert_eq!( + s.execute(Command::MemoryUsage("ghost".into())), + Value::BulkString(None) + ); + + // Equal-length key names, so the only difference the reply can reflect + // is the value. + s.execute(Command::Set("k1".into(), "x".into(), SetOptions::default())); + s.execute(Command::Set( + "k2".into(), + "x".repeat(4096).into(), + SetOptions::default(), + )); + + let (Value::Integer(small), Value::Integer(big)) = ( + s.execute(Command::MemoryUsage("k1".into())), + s.execute(Command::MemoryUsage("k2".into())), + ) else { + panic!("MEMORY USAGE should report an integer for a live key"); + }; + assert!(small > 0, "a stored key costs something"); + // Exact, not "bigger": the whole point of the reply is that the number + // moves with the value by the amount the value grew (4096 bytes minus + // the one byte the small key already held). + assert_eq!( + big - small, + 4095, + "the reported size must track the value: {small} vs {big}" + ); + } + + #[test] + fn memory_usage_agrees_with_what_eviction_bills_the_key() { + // The point of reusing `entry_size` rather than writing a second + // estimator: the answer to "what is this key costing me" and the number + // eviction acts on cannot drift apart. + let s = KeyValueStore::new(); + let empty = s.approximate_memory_bytes(); + s.execute(Command::HSet( + "h".into(), + vec![("f".into(), "y".repeat(1024).into())], + )); + let Value::Integer(reported) = s.execute(Command::MemoryUsage("h".into())) else { + panic!("expected an integer"); + }; + assert_eq!( + s.approximate_memory_bytes() - empty, + reported as usize, + "MEMORY USAGE must be the same measurement the eviction loop uses" + ); + } + + #[test] + fn memory_usage_treats_an_expired_key_as_absent() { + let s = KeyValueStore::new(); + s.execute(Command::Set("k".into(), "v".into(), SetOptions::default())); + s.execute(Command::PExpire("k".into(), 1)); + std::thread::sleep(std::time::Duration::from_millis(20)); + assert_eq!( + s.execute(Command::MemoryUsage("k".into())), + Value::BulkString(None), + "a key past its TTL is gone, and its footprint with it" + ); + } } #[cfg(test)] diff --git a/docs/server/commands.md b/docs/server/commands.md index 9bedb13..ed348c4 100644 --- a/docs/server/commands.md +++ b/docs/server/commands.md @@ -14,6 +14,8 @@ Recached implements the subset of RESP commands that most applications use. Comm | `CLIENT ` | Connection introspection. See [CLIENT](#client) below. | | `CONFIG GET parameter [parameter ...]` | Reports configuration parameters, matched by glob. See [CONFIG](#config) below. | | `COMMAND [COUNT\|LIST\|INFO\|DOCS]` | Reports the command catalog. See [COMMAND](#command) below. | +| `CLUSTER ` | Refused: Recached is standalone. The flag is in `INFO`. See [CLUSTER and MODULE](#cluster-and-module) below. | +| `MODULE LIST` | An empty array — there is no module API. See [CLUSTER and MODULE](#cluster-and-module) below. | --- @@ -65,6 +67,30 @@ Recached has no ACL system and no subcommand tree, so the ACL-categories, tips, --- +## CLUSTER and MODULE + +Recached is a single node with no module API. Both facts are reported the way Redis reports them, which for one of the two is not the obvious way. + +| Command | Description | +|---|---| +| `CLUSTER ` | Refused with `ERR This instance has cluster support disabled`. | +| `MODULE LIST` | An empty array. | +| `MODULE LOAD \| LOADEX \| UNLOAD` | Refused. | + +**Cluster support is advertised through `INFO`, not through `CLUSTER`.** A `redis-server` that was not started in cluster mode rejects the entire `CLUSTER` container with that exact sentence — it does *not* answer `CLUSTER INFO` with `cluster_enabled:0`, which is a common assumption and a wrong one. The flag lives in `INFO`'s `# Cluster` section, which is where every cluster-aware client actually reads it. Recached now emits that section, in the default set, so a bare `INFO` carries it: + +```bash +redis-cli -p 6379 INFO cluster +# Cluster +cluster_enabled:0 +``` + +Copying Redis's refusal verbatim matters more than it looks. Recached previously answered `ERR unknown command`, and that is the one reply a client cannot interpret: "unknown command" reads as *this server is too old to ask*, which is a different branch from *this server is not a cluster*. The sentence above puts a client on the same path it takes against the server it was written for. + +`MODULE LIST` returning an empty array is not a workaround — it is the same answer a stock `redis-server` with no modules loaded gives, and it lets a tool distinguish "no modules" from "cannot ask". `LOAD` and `UNLOAD` are refused rather than answered `+OK`, because an operator who believes a module loaded has a harder problem to debug than one who was told no. + +--- + ## INFO `INFO` reports the server's own state — uptime, client counts, memory, replication topology — in the same line format Redis uses, so `redis-cli info`, monitoring agents, and client library ready-checks all parse it unmodified. @@ -85,6 +111,7 @@ redis-cli -p 6379 INFO server memory | `persistence` | `loading`, `rdb_changes_since_last_save`, `rdb_last_save_time`, `rdb_bgsave_in_progress`, `aof_enabled` | | `stats` | `total_connections_received`, `total_commands_processed`, `keyspace_hits`, `keyspace_misses`, `evicted_keys` | | `replication` | `role`, `connected_slaves`, `connected_replicas`, `recached_replication_queue_depth`, `recached_replication_lag_frames` | +| `cluster` | `cluster_enabled:0` — always, see [CLUSTER and MODULE](#cluster-and-module) | | `keyspace` | `db0:keys=N,expires=N,avg_ttl=0` — omitted entirely when the keyspace is empty, as in Redis | | `recached` | `live_queries`, `watched_keys` — Recached-specific, no Redis equivalent | @@ -101,7 +128,9 @@ redis-cli -p 6379 INFO server memory ### Not implemented -`INFO` does not report the `cpu`, `commandstats`, `latencystats`, `cluster`, or `errorstats` sections. Per-command counters, error counts, and latency histograms are exported to [Prometheus](/server/operations#metrics-endpoint) on port 9091 instead, which is where they belong for dashboards and alerting. `INFO` is for the operator at a terminal and for client ready-checks. +`INFO` does not report the `cpu`, `commandstats`, `latencystats`, or `errorstats` sections. Per-command call counts and error counts are exported to [Prometheus](/server/operations#metrics-endpoint) on port 9091 instead, which is where they belong for dashboards and alerting. `INFO` is for the operator at a terminal and for client ready-checks. + +**Recached does not measure command latency anywhere** — not in `INFO`, and not on the metrics endpoint. `recached_commands_total` counts calls, `recached_command_errors_total` counts failures, and neither says how long anything took. So there is currently no way to see a slow command, which matters most for the commands that clone a whole collection under a shard guard: prefer the bounded reads (`HSCAN`, `SSCAN`, `ZSCAN`, `GETRANGE`) over `HGETALL` and `SMEMBERS` on large keys rather than expecting to catch the problem after the fact. Latency histograms and `SLOWLOG` are not implemented. ### Access @@ -161,6 +190,24 @@ The most common data type. Values are always stored as byte strings; numeric ope | `SCAN cursor [MATCH pattern] [COUNT count]` | Iterates keys incrementally, returning at most `COUNT` keys per call (default 10) plus the next cursor. Start with cursor `0` and continue until the returned cursor is `0`. `MATCH` filters results by glob pattern. As in Redis, keys inserted or deleted mid-iteration may be missed or returned twice. | | `DBSIZE` | Returns the total number of keys in the store. | | `FLUSHDB [ASYNC]` | Removes all keys from the store. `ASYNC` is accepted but does not change behavior (the flush is always synchronous). | +| `MEMORY USAGE key [SAMPLES count]` | Approximate bytes held by one key — the key name, its value, and a fixed per-entry overhead. Returns nil when the key does not exist or has expired. `SAMPLES` is accepted and ignored. | + +### MEMORY USAGE + +The figure is the same one the eviction loop bills the key for, not a second estimate written alongside it. That is the point: "which key is eating my `maxmemory`" and "which key gets evicted next" are answered from one measurement, so they cannot disagree. + +```bash +redis-cli -p 6379 MEMORY USAGE session:8f21 +(integer) 4162 +``` + +It counts the bytes Recached stores — key name, value contents, and 64 bytes of per-entry overhead — not the allocator's true footprint, which Recached does not manage and cannot see. Treat it as a way to compare keys against each other and to find the fat one, not as an exact resident-set contribution. + +Redis's `SAMPLES` bounds how much of a nested value it walks before extrapolating. Recached always walks all of it, so the option parses (a client that sends it is not broken) and the count is discarded. The reply is never less accurate than what was asked for. + +The other `MEMORY` subcommands — `DOCTOR`, `STATS`, `PURGE`, `MALLOC-STATS` — are refused. They describe an allocator arena that Recached has no equivalent of: it holds Rust values in a concurrent map and has nothing to defragment or free on demand. `INFO memory` reports what it can actually measure. + +`MEMORY USAGE` reads a key, so it is scoped like one: a WebSocket connection granted `cart:*` may measure `cart:42` and not `session:8f21`. See [Sync Scoping](/server/sync-scopes). --- @@ -456,6 +503,15 @@ Pub/Sub works over both TCP (port 6379) and WebSocket (port 6380). | `PSUBSCRIBE pattern [pattern ...]` | Subscribes to channels matching a glob pattern. `*` matches any sequence of bytes, `?` matches exactly one byte. **Character classes (`[abc]`) are not supported** — brackets match literally. Patterns are capped at 1,024 bytes. | | `PUNSUBSCRIBE [pattern ...]` | Unsubscribes from pattern subscriptions. With no arguments, unsubscribes from all patterns. | | `PUBLISH channel message` | Publishes a message to all subscribers of the given channel and all clients with matching pattern subscriptions. Returns the number of clients that received the message. | +| `PUBSUB CHANNELS [pattern]` | Channels with at least one subscriber. Without a pattern, all of them; with one, those whose name matches. Pattern subscriptions are never listed here — nobody is subscribed to a channel named `news.*`. | +| `PUBSUB NUMSUB [channel ...]` | Flat `[channel, count, channel, count, ...]`. A channel with no subscribers reports `0` rather than being dropped, so the reply can be read by position against the channels you asked about. Pattern subscribers are not counted; that is `NUMPAT`'s job. | +| `PUBSUB NUMPAT` | The number of **distinct** patterns under subscription. Two clients on `news.*` are one pattern, not two. | + +`PUBSUB` answers from the live subscriber registry, so it sees exactly what `PUBLISH` would deliver to. A channel disappears from `CHANNELS` when its last subscriber leaves — there is no lingering empty channel, because a channel is nothing more than its subscribers. + +`PUBSUB SHARDCHANNELS` and `PUBSUB SHARDNUMSUB` are refused. This is the one place these commands deliberately diverge from Redis, which answers both with an empty array even in standalone mode. It can afford to: `SSUBSCRIBE` and `SPUBLISH` work there, so an empty array honestly means "no shard channels are subscribed yet". Recached implements neither, so the same empty array would invite a client to call `SSUBSCRIBE` and fail. An error says what is true — the question does not apply here. + +`PUBSUB` enumerates what every other connection is subscribed to, so it is treated as an admin command and rejected on scope-limited WebSocket connections — the same line `KEYS` sits on. A scoped connection can still `SUBSCRIBE` and `PUBLISH` freely; channels are outside the scope system. Naming a channel and listing them all are different powers. ### Example diff --git a/server-native/src/main.rs b/server-native/src/main.rs index 4eb019f..74cf9b7 100644 --- a/server-native/src/main.rs +++ b/server-native/src/main.rs @@ -178,6 +178,10 @@ fn command_name(cmd: &Command) -> &'static str { Command::Client(_) => "client", Command::Config(_) => "config", Command::CommandQuery(_) => "command", + Command::Cluster(_) => "cluster", + Command::Module(_) => "module", + Command::PubSub(_) => "pubsub", + Command::Memory(_) | Command::MemoryUsage(_) => "memory", Command::Get(_) => "get", Command::ESet(_, _) => "eset", Command::Set(_, _, _) => "set", @@ -2132,6 +2136,14 @@ fn command_scope(cmd: &Command) -> CommandScope { | Command::Quit | Command::Client(_) | Command::CommandQuery(_) + // CLUSTER and MODULE answer the same sentence to everyone — "not a + // cluster", "no modules" — and describe no state a scope could protect. + | Command::Cluster(_) + | Command::Module(_) + // Every MEMORY subcommand other than USAGE is refused outright, so + // there is nothing here to scope either. USAGE reads a key and is + // classified with the key commands below. + | Command::Memory(_) | Command::Unknown(_) => CommandScope::KeyLess, Command::Keys(_) @@ -2148,6 +2160,13 @@ fn command_scope(cmd: &Command) -> CommandScope { // CONFIG reports server-wide limits and whether auth is on. Same // reasoning as INFO: not for a connection scoped to a few keys. | Command::Config(_) + // PUBSUB enumerates every channel every other client is subscribed to. + // A scoped connection can already SUBSCRIBE to any channel it can name + // — channels are outside the scope system entirely — but naming and + // listing are different powers, the same way GET is scoped and KEYS is + // Admin. NUMSUB and NUMPAT ride along rather than splitting the family + // across two scopes for one subcommand's worth of difference. + | Command::PubSub(_) | Command::ReplicaOfNoOne => CommandScope::Admin, Command::ESet(k, _) @@ -2172,6 +2191,7 @@ fn command_scope(cmd: &Command) -> CommandScope { | Command::PTtl(k) | Command::Persist(k) | Command::Type(k) + | Command::MemoryUsage(k) | Command::HSet(k, _) | Command::HGet(k, _) | Command::HGetAll(k) @@ -2381,6 +2401,44 @@ impl PubSubHub { self.pattern_subs.retain(|(_, id, _)| *id != conn_id); } + /// Channels with at least one live subscriber, for `PUBSUB CHANNELS`. + /// + /// `unsubscribe` and `unsubscribe_all` remove a channel's entry once its + /// last subscriber leaves, and `publish` drops senders whose receiver has + /// closed, so a key in `channel_subs` implies a live subscriber. The + /// `is_empty` guard covers the one window where it does not: a connection + /// that died between the last publish and its close handler. + fn active_channels(&self) -> impl Iterator { + self.channel_subs + .iter() + .filter(|(_, subs)| !subs.is_empty()) + .map(|(channel, _)| channel) + } + + /// Subscribers to one exact channel, for `PUBSUB NUMSUB`. Pattern + /// subscribers are deliberately not counted, matching Redis: a `PSUBSCRIBE` + /// is reported by `NUMPAT`, and counting it here would double-count a + /// client that holds both. + fn subscriber_count(&self, channel: &str) -> i64 { + self.channel_subs + .get(channel) + .map(|subs| subs.len() as i64) + .unwrap_or(0) + } + + /// Distinct patterns under subscription, for `PUBSUB NUMPAT`. Distinct is + /// the Redis definition: two clients on `news.*` are one pattern, not two. + fn pattern_count(&self) -> i64 { + let mut seen: Vec<&str> = self + .pattern_subs + .iter() + .map(|(p, _, _)| p.as_str()) + .collect(); + seen.sort_unstable(); + seen.dedup(); + seen.len() as i64 + } + /// Deliver to all matching subscribers; returns the count delivered. fn publish(&mut self, channel: &str, message: &[u8]) -> i64 { let mut count = 0i64; @@ -2814,6 +2872,7 @@ const DEFAULT_INFO_SECTIONS: &[&str] = &[ "persistence", "stats", "replication", + "cluster", "keyspace", "recached", ]; @@ -3102,6 +3161,129 @@ fn handle_config_command(args: &[String], facts: &ServerFacts, store: &KeyValueS } } +/// Handle `CLUSTER `. +/// +/// Recached does not cluster, and this reports that the way Redis does. A +/// `redis-server` that was not started in cluster mode does **not** answer +/// `CLUSTER INFO` with `cluster_enabled:0` — it rejects the whole `CLUSTER` +/// container with this exact sentence, and publishes the flag in `INFO`'s +/// `# Cluster` section instead. Copying the sentence rather than inventing a +/// slot map means a client's "am I clustered" branch takes the same path here +/// as against the server it was written for, and `ERR unknown command` (which +/// is what Recached said before) is the one answer that reads as "too old to +/// ask" rather than "not a cluster". +fn handle_cluster_command(_args: &[String]) -> Value { + Value::Error("ERR This instance has cluster support disabled".to_string()) +} + +/// Handle `MODULE `. +/// +/// There is no module API, so the loaded-module list is empty — which is a +/// real answer, and the same one a stock `redis-server` gives. `LOAD`, +/// `LOADEX` and `UNLOAD` are refused rather than answered `+OK`, because an +/// operator who believes a module loaded has a harder problem than one who +/// was told no. +fn handle_module_command(args: &[String]) -> Value { + match (args[0].to_uppercase().as_str(), args.len()) { + ("LIST", 1) => Value::Array(Some(vec![])), + ("HELP", 1) => Value::Array(Some( + [ + "MODULE ", + "LIST -- Return a list of loaded modules. Recached loads none.", + ] + .iter() + .map(|l| Value::SimpleString((*l).to_string())) + .collect(), + )), + _ => unknown_subcommand("MODULE", &args.join(" ")), + } +} + +/// Handle `PUBSUB [arg ...]` against the live subscriber hub. +/// +/// Recached has shipped `SUBSCRIBE`, `PSUBSCRIBE` and `PUBLISH` from the start +/// with no way to see any of it: `PUBLISH` returns a delivery count, and that +/// was the only observable. The hub already holds both registries, so these +/// three answers are a read of state that existed all along. +/// +/// `SHARDCHANNELS` and `SHARDNUMSUB` are refused rather than answered with the +/// empty array a standalone `redis-server` gives. This is a deliberate +/// divergence: Redis's empty array means "no shard channels are subscribed" on +/// a server where `SSUBSCRIBE` works, and a client reading it would reasonably +/// follow up with one. Recached has no `SSUBSCRIBE` or `SPUBLISH` at all, so +/// the honest answer is that the question does not apply here. +fn handle_pubsub_command(args: &[String], hub: &PubSubHub) -> Value { + match (args[0].to_uppercase().as_str(), args.len()) { + // No pattern means every active channel. Redis matches the pattern + // against channel names with the same globber it uses for keys, and so + // does this — `glob_match` is the one Recached already applies to + // PSUBSCRIBE, so a pattern selects here exactly what it would there. + ("CHANNELS", 1) => Value::Array(Some( + hub.active_channels() + .map(|c| Value::BulkString(Some(c.as_bytes().to_vec()))) + .collect(), + )), + ("CHANNELS", 2) => Value::Array(Some( + hub.active_channels() + .filter(|c| core_engine::store::glob_match(&args[1], c)) + .map(|c| Value::BulkString(Some(c.as_bytes().to_vec()))) + .collect(), + )), + // Flat [channel, count, channel, count, ...]. A channel nobody is + // subscribed to reports 0 rather than being dropped, so a caller that + // asked about N channels can index the reply by position. + ("NUMSUB", _) => { + let mut out = Vec::with_capacity((args.len() - 1) * 2); + for channel in &args[1..] { + out.push(Value::BulkString(Some(channel.as_bytes().to_vec()))); + out.push(Value::Integer(hub.subscriber_count(channel))); + } + Value::Array(Some(out)) + } + ("NUMPAT", 1) => Value::Integer(hub.pattern_count()), + ("HELP", 1) => Value::Array(Some( + [ + "PUBSUB ", + "CHANNELS [pattern] -- Return the currently active channels.", + "NUMSUB [channel ...] -- Return the subscriber count per channel.", + "NUMPAT -- Return the number of distinct subscribed patterns.", + ] + .iter() + .map(|l| Value::SimpleString((*l).to_string())) + .collect(), + )), + _ => unknown_subcommand("PUBSUB", &args.join(" ")), + } +} + +/// Handle `MEMORY ` for everything except `USAGE`, which is a key +/// read and goes to the store. +/// +/// `DOCTOR`, `STATS`, `PURGE` and `MALLOC-STATS` all describe an allocator +/// Recached does not manage — it holds Rust values in a `DashMap` and has no +/// arena to report on or free. Saying so beats a fabricated report. +fn handle_memory_command(args: &[String]) -> Value { + match (args[0].to_uppercase().as_str(), args.len()) { + ("HELP", 1) => Value::Array(Some( + [ + "MEMORY ", + "USAGE [SAMPLES ] -- Bytes held by one key. SAMPLES is accepted \ + and ignored: the estimate always covers every element.", + ] + .iter() + .map(|l| Value::SimpleString((*l).to_string())) + .collect(), + )), + ("DOCTOR" | "STATS" | "PURGE" | "MALLOC-STATS", 1) => Value::Error(format!( + "ERR MEMORY {} is not supported: Recached does not manage its own allocator, \ + so it has nothing to report or free. MEMORY USAGE and INFO memory are the \ + measurements it can make.", + args[0].to_uppercase() + )), + _ => unknown_subcommand("MEMORY", &args.join(" ")), + } +} + /// `COMMAND INFO`'s per-command reply: name, arity, flags, key positions. fn command_info_entry(spec: &catalog::CommandSpec) -> Value { Value::Array(Some(vec![ @@ -3375,6 +3557,13 @@ fn render_info( ) } } + // How a cluster-aware client actually learns it is talking to a + // single node. `CLUSTER INFO` is not that channel: a `redis-server` + // built for standalone answers it with an error, not with + // `cluster_enabled:0`, so this line is the only place the answer + // exists. Reporting it costs one line and stops a client from + // guessing. + "cluster" => "cluster_enabled:0\r\n".to_string(), // Recached-specific: the live-query machinery has no Redis analogue, // so it gets its own section rather than being smuggled into one. "recached" => { @@ -5087,6 +5276,26 @@ async fn handle_tcp( if writer.write_all(&resp).await.is_err() { break 'outer; } continue 'parse; } + Command::Cluster(args) => { + let resp = handle_cluster_command(args).serialize(); + if writer.write_all(&resp).await.is_err() { break 'outer; } + continue 'parse; + } + Command::Module(args) => { + let resp = handle_module_command(args).serialize(); + if writer.write_all(&resp).await.is_err() { break 'outer; } + continue 'parse; + } + Command::Memory(args) => { + let resp = handle_memory_command(args).serialize(); + if writer.write_all(&resp).await.is_err() { break 'outer; } + continue 'parse; + } + Command::PubSub(args) => { + let resp = handle_pubsub_command(args, &*pubsub.lock().await).serialize(); + if writer.write_all(&resp).await.is_err() { break 'outer; } + continue 'parse; + } Command::ReplicaOfNoOne => { state.promote_to_primary(); if writer.write_all(b"+OK\r\n").await.is_err() { break 'outer; } @@ -5770,6 +5979,22 @@ async fn handle_ws( ws_send!(&handle_command_query(args, 3).serialize()); continue 'outer; } + Command::Cluster(args) => { + ws_send!(&handle_cluster_command(args).serialize()); + continue 'outer; + } + Command::Module(args) => { + ws_send!(&handle_module_command(args).serialize()); + continue 'outer; + } + Command::Memory(args) => { + ws_send!(&handle_memory_command(args).serialize()); + continue 'outer; + } + Command::PubSub(args) => { + ws_send!(&handle_pubsub_command(args, &*pubsub.lock().await).serialize()); + continue 'outer; + } Command::ReplicaOfNoOne => { state.promote_to_primary(); ws_send!(b"+OK\r\n"); @@ -7301,6 +7526,44 @@ mod tests { ); } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn integration_ws_reaches_the_introspection_commands_too() { + // `handle_tcp` and `handle_ws` are two hand-maintained copies of one + // command loop, so the standing hazard when adding a server-level + // command is wiring it into one and not the other — which compiles, and + // fails only over the transport nobody checked. Every command added + // outside the store belongs in a test like this one. + let srv = spawn_ws_server().await; + let mut c = WsClient::connect(srv.tcp_addr).await; + + assert_eq!(c.cmd(&["SET", "k", "hello"]).await, ok()); + let usage = c.cmd(&["MEMORY", "USAGE", "k"]).await; + assert!( + matches!(usage, Value::Integer(n) if n > 0), + "MEMORY USAGE over WS: {usage:?}" + ); + assert_eq!( + c.cmd(&["MEMORY", "USAGE", "ghost"]).await, + Value::BulkString(None) + ); + assert!(matches!( + c.cmd(&["MEMORY", "DOCTOR"]).await, + Value::Error(_) + )); + + assert_eq!(c.cmd(&["MODULE", "LIST"]).await, Value::Array(Some(vec![]))); + assert!(matches!(c.cmd(&["CLUSTER", "INFO"]).await, Value::Error(_))); + + // No subscribers on this connection, so the registry is empty — the + // point is that the command is answered at all rather than falling + // through to the store's "handled by the connection layer" refusal. + assert_eq!( + c.cmd(&["PUBSUB", "CHANNELS"]).await, + Value::Array(Some(vec![])) + ); + assert_eq!(c.cmd(&["PUBSUB", "NUMPAT"]).await, Value::Integer(0)); + } + #[tokio::test] async fn replication_lag_counts_unacknowledged_frames() { // A replica that receives frames but never acknowledges them is exactly @@ -8498,6 +8761,11 @@ mod tests { Expect::Admin, ), (Command::CommandQuery(vec![]), Expect::KeyLess), + (Command::Cluster(vec!["INFO".into()]), Expect::KeyLess), + (Command::Module(vec!["LIST".into()]), Expect::KeyLess), + (Command::Memory(vec!["DOCTOR".into()]), Expect::KeyLess), + (Command::MemoryUsage("k".into()), Expect::Keys(&["k"])), + (Command::PubSub(vec!["CHANNELS".into()]), Expect::Admin), (Command::Unknown("X".into()), Expect::KeyLess), ] } @@ -8925,6 +9193,224 @@ mod tests { } } + // ── Introspection: PUBSUB / CLUSTER / MODULE / MEMORY ───────────────────── + + /// A hub with `channels` subscribed and `patterns` psubscribed. The senders + /// are kept alive by the returned vector — dropping them would close the + /// receivers and make the hub look empty. + fn hub_with( + channels: &[(u64, &str)], + patterns: &[(u64, &str)], + ) -> (PubSubHub, Vec>) { + let mut hub = PubSubHub::new(); + let mut keepalive = Vec::new(); + for (id, ch) in channels { + let (tx, rx) = mpsc::unbounded_channel(); + hub.subscribe(*id, ch, tx); + keepalive.push(rx); + } + for (id, pat) in patterns { + let (tx, rx) = mpsc::unbounded_channel(); + hub.psubscribe(*id, pat, tx); + keepalive.push(rx); + } + (hub, keepalive) + } + + fn bulk_strings(v: &Value) -> Vec { + match v { + Value::Array(Some(items)) => items + .iter() + .map(|i| match i { + Value::BulkString(Some(b)) => String::from_utf8_lossy(b).into_owned(), + other => panic!("expected a bulk string, got {other:?}"), + }) + .collect(), + other => panic!("expected an array, got {other:?}"), + } + } + + #[test] + fn pubsub_channels_lists_only_channels_with_subscribers() { + let (hub, _keep) = hub_with(&[(1, "news"), (2, "news"), (3, "sports")], &[(4, "news.*")]); + + let mut all = bulk_strings(&handle_pubsub_command(&["CHANNELS".into()], &hub)); + all.sort(); + assert_eq!(all, vec!["news".to_string(), "sports".to_string()]); + + // A pattern subscriber is not a channel. Redis reports `news.*` under + // NUMPAT and never under CHANNELS, because nobody is subscribed to a + // channel by that name. + assert!(!all.contains(&"news.*".to_string())); + + let filtered = bulk_strings(&handle_pubsub_command( + &["CHANNELS".into(), "spo*".into()], + &hub, + )); + assert_eq!(filtered, vec!["sports".to_string()]); + } + + #[test] + fn pubsub_numsub_counts_per_channel_and_keeps_the_caller_s_order() { + let (hub, _keep) = hub_with(&[(1, "news"), (2, "news"), (3, "sports")], &[(4, "news.*")]); + + let reply = handle_pubsub_command( + &[ + "NUMSUB".into(), + "sports".into(), + "news".into(), + "nobody-here".into(), + ], + &hub, + ); + assert_eq!( + reply, + Value::Array(Some(vec![ + Value::BulkString(Some(b"sports".to_vec())), + Value::Integer(1), + Value::BulkString(Some(b"news".to_vec())), + // Two subscribers, and the `news.*` pattern subscriber is not + // one of them: NUMPAT's job, counted here would be double. + Value::Integer(2), + Value::BulkString(Some(b"nobody-here".to_vec())), + // Present with a zero rather than omitted, so a caller can read + // the reply by position against the channels it asked about. + Value::Integer(0), + ])) + ); + + // No channels named is a legal call and an empty reply, not an error. + assert_eq!( + handle_pubsub_command(&["NUMSUB".into()], &hub), + Value::Array(Some(vec![])) + ); + } + + #[test] + fn pubsub_numpat_counts_distinct_patterns_not_subscribers() { + let (hub, _keep) = hub_with(&[], &[(1, "news.*"), (2, "news.*"), (3, "sports.*")]); + assert_eq!( + handle_pubsub_command(&["NUMPAT".into()], &hub), + Value::Integer(2), + "two clients on one pattern are one pattern" + ); + } + + #[test] + fn pubsub_channels_forgets_a_channel_once_its_last_subscriber_leaves() { + let (mut hub, _keep) = hub_with(&[(1, "news"), (2, "news")], &[]); + hub.unsubscribe(1, "news"); + assert_eq!( + bulk_strings(&handle_pubsub_command(&["CHANNELS".into()], &hub)), + vec!["news".to_string()] + ); + hub.unsubscribe(2, "news"); + assert!( + bulk_strings(&handle_pubsub_command(&["CHANNELS".into()], &hub)).is_empty(), + "an abandoned channel is not an active channel" + ); + } + + #[test] + fn pubsub_refuses_the_sharded_subcommands() { + let (hub, _keep) = hub_with(&[], &[]); + // A standalone redis-server answers these with an empty array, and this + // is the one place Recached deliberately does not match it: there, the + // empty array sits next to a working SSUBSCRIBE. Here there is none, so + // "no shard channels are subscribed" would invite a call that fails. + for sub in ["SHARDCHANNELS", "SHARDNUMSUB"] { + assert!( + matches!( + handle_pubsub_command(&[sub.to_string()], &hub), + Value::Error(_) + ), + "{sub} should be refused" + ); + } + } + + #[test] + fn cluster_is_refused_the_way_a_standalone_redis_refuses_it() { + // Verified against redis-server 7.2.5: a server not started in cluster + // mode rejects the whole CLUSTER container with this sentence. It does + // *not* answer INFO with cluster_enabled:0 — that lives in `INFO`. + for sub in ["INFO", "NODES", "SLOTS", "MYID", "SHARDS"] { + assert_eq!( + handle_cluster_command(&[sub.to_string()]), + Value::Error("ERR This instance has cluster support disabled".to_string()), + "CLUSTER {sub}" + ); + } + } + + #[test] + fn info_publishes_the_cluster_flag_that_cluster_info_cannot() { + let store = KeyValueStore::new(); + let body = render_info( + &["cluster".to_string()], + server_facts(), + &store, + sampled_keyspace(&store), + false, + ReplInfo::default(), + 0, + 0, + 0, + ); + assert!(body.contains("# Cluster\r\n"), "section header: {body:?}"); + assert!(body.contains("cluster_enabled:0"), "{body:?}"); + + // And it is in the default set, so a client that sends a bare INFO — + // which is what every cluster-aware client actually sends — sees it. + let default = render_info( + &[], + server_facts(), + &store, + sampled_keyspace(&store), + false, + ReplInfo::default(), + 0, + 0, + 0, + ); + assert!(default.contains("cluster_enabled:0"), "{default:?}"); + } + + #[test] + fn module_list_is_empty_and_loading_is_refused() { + assert_eq!( + handle_module_command(&["LIST".to_string()]), + Value::Array(Some(vec![])), + "no modules is an answer, not an error" + ); + for sub in ["LOAD", "LOADEX", "UNLOAD"] { + assert!( + matches!( + handle_module_command(&[sub.to_string(), "/tmp/x.so".to_string()]), + Value::Error(_) + ), + "MODULE {sub} should be refused rather than answered +OK" + ); + } + } + + #[test] + fn memory_allocator_subcommands_are_refused_with_a_reason() { + for sub in ["DOCTOR", "STATS", "PURGE", "MALLOC-STATS"] { + let Value::Error(msg) = handle_memory_command(&[sub.to_string()]) else { + panic!("MEMORY {sub} should be refused"); + }; + assert!( + msg.contains("MEMORY USAGE"), + "the refusal should name what does work: {msg}" + ); + } + assert!(matches!( + handle_memory_command(&["HELP".to_string()]), + Value::Array(Some(_)) + )); + } + // ── Wire encoding ───────────────────────────────────────────────────────── #[test] From 86effd8bf9a70cdc7efe0ad326d1d99f506d5ee5 Mon Sep 17 00:00:00 2001 From: Dennis Paler Date: Mon, 3 Aug 2026 17:17:25 +0800 Subject: [PATCH 3/3] fix(server): propagate expiries as absolute deadlines so a restart stops resurrecting keys that died during it --- .github/workflows/ci.yml | 4 +- .github/workflows/release.yml | 4 +- CHANGELOG.md | 91 +- Cargo.lock | 54 +- Cargo.toml | 2 +- Dockerfile | 6 +- Formula/recached.rb | 6 +- README.md | 3 + core-engine/src/store.rs | 127 +- docs/server/configuration.md | 4 +- sdks/recached-react/package.json | 2 +- sdks/recached-vue/package.json | 2 +- server-native/Cargo.toml | 15 +- server-native/src/clients.rs | 372 + server-native/src/config.rs | 600 ++ server-native/src/connection.rs | 1528 ++++ server-native/src/info.rs | 749 ++ server-native/src/main.rs | 11870 ++-------------------------- server-native/src/persistence.rs | 457 ++ server-native/src/propagation.rs | 1214 +++ server-native/src/pubsub.rs | 192 + server-native/src/replication.rs | 661 ++ server-native/src/server_state.rs | 185 + server-native/src/sync_scopes.rs | 318 + server-native/src/tests.rs | 5108 ++++++++++++ server-native/src/tls.rs | 277 + server-native/src/watch.rs | 96 + wasm-edge/package.json | 2 +- 28 files changed, 12761 insertions(+), 11188 deletions(-) create mode 100644 server-native/src/clients.rs create mode 100644 server-native/src/config.rs create mode 100644 server-native/src/connection.rs create mode 100644 server-native/src/info.rs create mode 100644 server-native/src/persistence.rs create mode 100644 server-native/src/propagation.rs create mode 100644 server-native/src/pubsub.rs create mode 100644 server-native/src/replication.rs create mode 100644 server-native/src/server_state.rs create mode 100644 server-native/src/sync_scopes.rs create mode 100644 server-native/src/tests.rs create mode 100644 server-native/src/tls.rs create mode 100644 server-native/src/watch.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 22ffa6d..04ed632 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,7 +59,7 @@ jobs: # functions (scope classification, wire encoding, config parsing), not to # chase a number the I/O layer cannot reach. - name: Enforce server-native coverage floor (65%) - run: cargo llvm-cov -p server-native --summary-only --fail-under-lines 65 -- --include-ignored + run: cargo llvm-cov -p recached --summary-only --fail-under-lines 65 -- --include-ignored # --------------------------------------------------------------------------- # Browser tests — the IndexedDB persistence layer and the engine's behaviour @@ -119,7 +119,7 @@ jobs: shared-key: "recached-ci-cache" - name: Run load & chaos tests - run: cargo test -p server-native -- --include-ignored + run: cargo test -p recached -- --include-ignored timeout-minutes: 5 typecheck-js: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index eb7a690..664bfc5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -116,13 +116,13 @@ jobs: shared-key: "recached-release-${{ matrix.target }}" - name: Build release binary - run: cargo build --release --package server-native --target ${{ matrix.target }} + run: cargo build --release --package recached --target ${{ matrix.target }} - name: Upload binary to GitHub Release uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: target/${{ matrix.target }}/release/server-native${{ matrix.binary_suffix || '' }} + file: target/${{ matrix.target }}/release/recached-server${{ matrix.binary_suffix || '' }} asset_name: ${{ matrix.artifact }} tag: ${{ github.ref }} overwrite: true diff --git a/CHANGELOG.md b/CHANGELOG.md index 956e2ae..d110713 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to Recached are documented here. --- -## [Unreleased] +## [0.2.5] — Unreleased ### Added @@ -56,8 +56,97 @@ All notable changes to Recached are documented here. to a working `SSUBSCRIBE`, and Recached implements neither `SSUBSCRIBE` nor `SPUBLISH`, so the same reply would invite a follow-up call that fails. +### Added + +- **`RECACHED_PORT` and `RECACHED_WS_PORT`.** The RESP and WebSocket ports were compiled in, so two + instances could not share a host — a replica beside its primary was impossible — and there was no + way off 6379, the first port any commodity scanner probes. Defaults are unchanged, so no existing + deployment needs to do anything. The port is not itself a security control (`RECACHED_BIND`, the + password, TLS and the allowlists are), which is why exposing it is safe; what is *not* safe is a + typo falling back to the default, so an unparseable value, `0`, or the two ports being equal is a + startup error rather than a silent 6379. Ports below 1024 still require root, enforced by the OS. + The startup warnings about the sync port now name the port actually in use instead of a hardcoded + 6380, and `INFO`'s `tcp_port`/`recached_ws_port` report the real values. + ### Fixed +- **`RECACHED_METRICS_PORT=0` now disables the exporter, as the reference has always said it did.** + `0` parsed fine and `bind("host:0")` hands the listener an OS-assigned ephemeral port, so an + operator switching metrics *off* got them served on an unpredictable port instead — the opposite + of the request, and unlikely to be noticed until something scraped it. A collision on this port + also aborted startup with a bare `panic!` and a backtrace note, which reads like a bug in Recached + rather than two servers wanting one port; it is now an error that names the conflict and points at + the three port variables. An unparseable value is a startup error. + +- **`INCR` and friends no longer clear a key's TTL on the replica, in the AOF, or in synced + browsers.** Counters propagate by value — `SET key ` — so that a replica which missed a + frame converges on the primary's number instead of compounding its own. But a bare `SET` also + clears the expiry, and Redis's `INCR` leaves it untouched, so the single most common expiring + counter idiom — `INCR key` followed by `EXPIRE key window` — replayed as a key with no expiry at + all. The rate-limit bucket, the per-minute quota and the retry counter each became permanent + everywhere but on the primary, and the window never reset because the key it keyed on never went + away. They now propagate as `SET KEEPTTL`, which keeps the by-value convergence while + leaving the deadline where the primary has it. `GETSET` still propagates a bare `SET`, because + `GETSET` really does clear the TTL. + +- **`TTL` rounds to the nearest second instead of truncating.** `SET k v EX 100` followed immediately + by `TTL k` answered 99: the microseconds spent between the two commands took the remainder just + below 100 000 ms, and integer division discarded the rest. Every reading was up to a second short, + which breaks a ported test suite asserting the value it just set and makes any client that renews + below a threshold renew early on every pass. Now `(remaining_ms + 500) / 1000`, matching Redis. + `PTTL` is unchanged — it reports milliseconds and has nothing to round. + +- **`PUBLISH` may be used inside `MULTI`.** Redis allows it, and announcing a change atomically with + the write that caused it is an ordinary reason to open a transaction at all, but Recached refused + it alongside `SUBSCRIBE` and `WATCH`. Simply letting it queue would have been worse than the + refusal: delivery lives in the connection loop and the store's `PUBLISH` is a stub that answers 0 + and sends nothing, so the message would have been swallowed silently and `EXEC` would have reported + a plausible zero. `EXEC` now dispatches queued publishes to the subscriber hub itself, so the reply + is the real delivery count and subscribers actually receive the message. `SUBSCRIBE`, `PSUBSCRIBE` + and `WATCH` remain unqueueable, which is correct. + +- **The binary is `recached-server` and the crate is `recached`, as the docs have always claimed.** + The package was `server-native` and produced a `server-native` binary, so `cargo install recached` + installed nothing — the name was not even registered — and `cargo build --bin recached-server`, + the command in the contributing guide, failed outright. Only Docker and Homebrew worked, because + both rename the artefact as they copy it. The directory keeps its name; it describes the role, + while the package describes the product. CI, the release workflow and the Dockerfile follow. + +- **Expiries now propagate as absolute deadlines, so a restart no longer resurrects a key that + should have died during it.** Every write leaves the server as one RESP frame that the AOF, the + replication log and the browser sync fan-out all consume, and a relative TTL (`SET k v PX 5000`, + `PEXPIRE k 5000`) was written into that frame verbatim. Each consumer then re-based the deadline + onto *its own* clock at *its own* arrival time, so the key's lifetime silently restarted on every + hop. At AOF replay this was total: a key written with `EX 5` and replayed an hour later came back + alive with a fresh five seconds — a revoked session, an abandoned lock, a spent idempotency key or + a rate-limit window, all restored by the restart that was supposed to be transparent. Replicas + expired their copy later than the primary by the replication delay, and a reconnecting browser + reset the TTL of every key the sync socket replayed to it. Relative expiries are now converted + against the propagation timestamp and travel as `PXAT`/`PEXPIREAT`, which is what Redis does and + what the already-correct `EXAT`/`PXAT`/`EXPIREAT`/`PEXPIREAT` arms beside them already did. An + absolute deadline is idempotent under replay: applying it once or a thousand times, now or after + an hour of downtime, names the same instant, and one already in the past needs no special case — + the store reads such an entry as expired and the sweeper reaps it. The snapshot path was never + affected; it has always stored absolute expiries, which is why `SAVE` and the AOF disagreed about + whether a key still existed. + +- **`EXEC` no longer runs the rest of a transaction after a command failed to queue.** Redis + refuses an unrecognised verb at queue time and poisons the transaction, so `EXEC` replies + `EXECABORT Transaction discarded because of previous errors.` and runs nothing. Recached parses an + unrecognised verb into an internal `Unknown` command, which queued happily behind a `+QUEUED` and + only errored while executing — leaving every *other* command in the transaction applied. On a + server that implements a deliberate subset of Redis that is a live hazard rather than a corner + case: `MULTI; ZPOPMIN q; LPUSH processing x; EXEC` pushed onto `processing` without ever popping + `q`, silently, and MULTI is exactly the construct a caller reaches for to prevent that. Anything + that fails to queue now sets the abort flag — an unknown verb, a frame that will not parse (bad + arity, malformed argument), a command not allowed inside a transaction, and a queue over + `RECACHED_MAX_MULTI_QUEUE` — and the unknown-verb rejection is delivered at queue time with the + same wording the store gives outside a transaction. `MULTI` and `DISCARD` clear the flag, so a + poisoned transaction does not wedge later ones on the same connection. The CAS abort is + deliberately left distinct: a `WATCH` conflict still replies with a nil array, because "retry me" + and "fix your request" are different answers and a retry loop must be able to tell them apart. + Fixed on both the TCP and WebSocket command paths. + - **The command reference no longer claims Recached exports latency histograms.** `INFO`'s "not implemented" note said per-command latency was on the Prometheus endpoint instead — it is not, and never was. Recached has no latency instrumentation at all: `recached_commands_total` counts diff --git a/Cargo.lock b/Cargo.lock index e5f7b66..a0f444b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -124,7 +124,7 @@ dependencies = [ [[package]] name = "core-engine" -version = "0.2.4" +version = "0.2.5" dependencies = [ "dashmap", "indexmap", @@ -853,6 +853,30 @@ dependencies = [ "bitflags", ] +[[package]] +name = "recached" +version = "0.2.5" +dependencies = [ + "base64", + "core-engine", + "futures-util", + "hmac", + "metrics", + "metrics-exporter-prometheus", + "num_cpus", + "rand", + "rmp-serde", + "serde", + "sha2", + "socket2 0.5.10", + "tikv-jemallocator", + "tokio", + "tokio-rustls", + "tokio-tungstenite", + "tracing", + "tracing-subscriber", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -1056,30 +1080,6 @@ dependencies = [ "zmij", ] -[[package]] -name = "server-native" -version = "0.2.4" -dependencies = [ - "base64", - "core-engine", - "futures-util", - "hmac", - "metrics", - "metrics-exporter-prometheus", - "num_cpus", - "rand", - "rmp-serde", - "serde", - "sha2", - "socket2 0.5.10", - "tikv-jemallocator", - "tokio", - "tokio-rustls", - "tokio-tungstenite", - "tracing", - "tracing-subscriber", -] - [[package]] name = "sha1" version = "0.10.6" @@ -1195,7 +1195,7 @@ dependencies = [ [[package]] name = "sync-client" -version = "0.2.4" +version = "0.2.5" dependencies = [ "core-engine", ] @@ -1581,7 +1581,7 @@ checksum = "60238e5b4b1b295701d6f9a66d2a126fe19990348f5fb9dae3b623a370119d94" [[package]] name = "wasm-edge" -version = "0.2.4" +version = "0.2.5" dependencies = [ "core-engine", "getrandom 0.3.4", diff --git a/Cargo.toml b/Cargo.toml index 1efa3d2..4d157cb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ resolver = "2" # ── Single source of truth for all crate versions ──────────────────────────── # Members inherit with: version.workspace = true / edition.workspace = true [workspace.package] -version = "0.2.4" +version = "0.2.5" edition = "2024" license = "Apache-2.0" authors = ["ThinkGrid Labs"] diff --git a/Dockerfile b/Dockerfile index b048a48..90cbbd9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,7 +25,7 @@ RUN mkdir -p core-engine/src server-native/src sync-client/src wasm-edge/src && echo "" > core-engine/src/lib.rs && \ echo "" > sync-client/src/lib.rs && \ echo "" > wasm-edge/src/lib.rs && \ - cargo build --release --package server-native && \ + cargo build --release --package recached && \ rm -rf core-engine/src server-native/src # Now copy real source and do the real build (only changed crates recompile). @@ -36,7 +36,7 @@ COPY server-native/src server-native/src # above — cargo would then consider the crates unchanged and ship the dummy # binary. Touch the roots so both crates always recompile. RUN touch core-engine/src/lib.rs server-native/src/main.rs && \ - cargo build --release --package server-native + cargo build --release --package recached # ── Stage 2: Runtime ───────────────────────────────────────────────────────── FROM debian:bookworm-slim @@ -45,7 +45,7 @@ RUN apt-get update && \ apt-get install -y --no-install-recommends ca-certificates && \ rm -rf /var/lib/apt/lists/* -COPY --from=builder /app/target/release/server-native /usr/local/bin/recached-server +COPY --from=builder /app/target/release/recached-server /usr/local/bin/recached-server EXPOSE 6379 EXPOSE 6380 diff --git a/Formula/recached.rb b/Formula/recached.rb index 518c5d8..ebda0dd 100644 --- a/Formula/recached.rb +++ b/Formula/recached.rb @@ -1,7 +1,7 @@ class Recached < Formula desc "Blazing fast, multi-core drop-in replacement for Redis" homepage "https://github.com/recached-dev/recached" - version "0.2.4" + version "0.2.5" license "Apache-2.0" # The checksums below are placeholders until the v0.2.4 release artifacts @@ -16,11 +16,11 @@ class Recached < Formula # placeholder makes brew fail loudly, which is the far better failure. on_macos do on_intel do - url "https://github.com/recached-dev/recached/releases/download/v0.2.4/recached-macos-amd64" + url "https://github.com/recached-dev/recached/releases/download/v0.2.5/recached-macos-amd64" sha256 "REPLACE_WITH_AMD64_SHA256" end on_arm do - url "https://github.com/recached-dev/recached/releases/download/v0.2.4/recached-macos-arm64" + url "https://github.com/recached-dev/recached/releases/download/v0.2.5/recached-macos-arm64" sha256 "REPLACE_WITH_ARM64_SHA256" end end diff --git a/README.md b/README.md index a94f751..1cb1971 100644 --- a/README.md +++ b/README.md @@ -97,6 +97,9 @@ port beyond localhost, work through [recached.dev/server/security](https://recached.dev/server/security): a default server has no password, no TLS, and no restriction on which web pages may open the sync socket. +`6379` and `6380` are defaults, not fixtures — set `RECACHED_PORT` and `RECACHED_WS_PORT` (plus +`RECACHED_METRICS_PORT`) to move them, which is also what running two instances on one host takes. + --- ## Benchmarks diff --git a/core-engine/src/store.rs b/core-engine/src/store.rs index 64ebab3..d711a19 100644 --- a/core-engine/src/store.rs +++ b/core-engine/src/store.rs @@ -1782,7 +1782,19 @@ impl KeyValueStore { Some(e) if e.is_expired(now) => Value::Integer(-2), Some(e) => match e.expires_at_ms { None => Value::Integer(-1), - Some(exp) => Value::Integer((exp.saturating_sub(now) / 1000) as i64), + // Rounded to nearest, not truncated, matching Redis. + // Truncating meant `SET k v EX 100` followed + // immediately by `TTL k` answered 99: the handful of + // microseconds spent between the two commands took the + // remainder just below 100_000 ms, and `/ 1000` threw + // the rest away. Every TTL read was up to a second + // short, which breaks ported test suites asserting the + // value they just set and makes any client that renews + // at a threshold renew early, forever. + Some(exp) => { + let remaining_ms = exp.saturating_sub(now); + Value::Integer((remaining_ms.saturating_add(500) / 1000) as i64) + } }, } } @@ -8266,3 +8278,116 @@ mod zset_index_tests { ); } } + +// ── TTL rounding ────────────────────────────────────────────────────────────── + +/// `TTL` reports whole seconds rounded to nearest, as Redis does. +/// +/// It used to truncate, so `SET k v EX 100` followed immediately by `TTL k` +/// answered 99: the microseconds between the two commands took the remainder +/// just below 100_000 ms and `/ 1000` discarded the rest. Every reading was up +/// to a second short, which breaks a ported test suite asserting the value it +/// just set, and makes a client that renews below a threshold renew early on +/// every pass. `PTTL` is unaffected — it reports milliseconds and has nothing +/// to round. +#[cfg(test)] +mod ttl_rounding_tests { + use super::*; + use crate::cmd::{SetExpiry, SetOptions}; + + fn store_with_expiry_in(ms: u64) -> KeyValueStore { + let s = KeyValueStore::new(); + s.execute(Command::Set( + "k".into(), + "v".into(), + SetOptions { + expiry: Some(SetExpiry::Px(ms)), + ..Default::default() + }, + )); + s + } + + fn ttl(s: &KeyValueStore) -> i64 { + match s.execute(Command::Ttl("k".into())) { + Value::Integer(n) => n, + other => panic!("expected an integer, got {other:?}"), + } + } + + #[test] + fn a_freshly_set_ttl_reads_back_as_the_value_that_was_set() { + // The regression that motivated this: the number a caller just wrote + // must be the number they read back. + // + // The sleep is load-bearing. In-process, SET and TTL can land in the + // same millisecond, and with a remainder of exactly `secs * 1000` even + // truncation answers correctly — so without a gap this test passes + // against the bug it exists to catch. A single millisecond of real + // elapsed time is what separates the two: truncation then answers + // `secs - 1`, rounding still answers `secs`. Over TCP the gap is always + // there, which is why the bug showed up against a live server first. + for secs in [1u64, 10, 100, 3600] { + let s = KeyValueStore::new(); + s.execute(Command::Set( + "k".into(), + "v".into(), + SetOptions { + expiry: Some(SetExpiry::Ex(secs)), + ..Default::default() + }, + )); + std::thread::sleep(std::time::Duration::from_millis(2)); + assert_eq!( + ttl(&s), + secs as i64, + "SET k v EX {secs} then TTL k must answer {secs}" + ); + } + } + + #[test] + fn remainders_round_to_the_nearest_second() { + // 1600 ms is nearer 2 s than 1 s; 1400 ms is nearer 1 s. Truncation + // answered 1 for both. + assert_eq!(ttl(&store_with_expiry_in(1_600)), 2); + assert_eq!(ttl(&store_with_expiry_in(1_400)), 1); + // Exactly half a second rounds up, matching Redis's `(ttl + 500) / 1000`. + assert_eq!(ttl(&store_with_expiry_in(2_500)), 3); + assert_eq!(ttl(&store_with_expiry_in(600)), 1); + } + + #[test] + fn a_sub_second_remainder_still_reports_a_live_key_not_zero_or_minus_two() { + // A key with 400 ms left is alive. Reporting -2 would say "no such key" + // and 0 is only correct once it is nearly gone. + let s = store_with_expiry_in(400); + assert_eq!(ttl(&s), 0, "400 ms rounds down to 0 whole seconds"); + assert_eq!( + s.execute(Command::Get("k".into())), + Value::BulkString(Some(b"v".to_vec())), + "the key is still readable while TTL reports 0" + ); + } + + #[test] + fn the_sentinels_are_unchanged() { + let s = KeyValueStore::new(); + assert_eq!(s.execute(Command::Ttl("ghost".into())), Value::Integer(-2)); + s.execute(Command::Set("k".into(), "v".into(), SetOptions::default())); + assert_eq!(s.execute(Command::Ttl("k".into())), Value::Integer(-1)); + } + + #[test] + fn pttl_still_reports_exact_milliseconds() { + // Rounding belongs to TTL alone; PTTL must not gain a half-second bias. + let s = store_with_expiry_in(1_600); + match s.execute(Command::PTtl("k".into())) { + Value::Integer(ms) => assert!( + (1_400..=1_600).contains(&ms), + "PTTL should be ~1600 ms, got {ms}" + ), + other => panic!("expected an integer, got {other:?}"), + } + } +} diff --git a/docs/server/configuration.md b/docs/server/configuration.md index fff74e2..a05d0c1 100644 --- a/docs/server/configuration.md +++ b/docs/server/configuration.md @@ -14,7 +14,9 @@ Recached is configured entirely through environment variables. There is no confi | `RECACHED_SYNC_SECRET` | _(none)_ | Enables **strict sync scoping** on the WebSocket port: clients receive no mutation pushes and may run no key commands until they present a signed scope token (`SYNC TOKEN `), and are then restricted to the keys their token grants. Without it, every WebSocket client receives every mutation. See [Sync Scopes](/server/sync-scopes). | | `RECACHED_MAX_KEYS` | _(unlimited)_ | Maximum number of keys in the store. When this limit is reached, behavior depends on `RECACHED_EVICTION`. If set to `noeviction` (the default), write commands that would exceed the cap return an error. | | `RECACHED_EVICTION` | `noeviction` | Eviction policy when `RECACHED_MAX_KEYS` is reached. See eviction policies below. | -| `RECACHED_METRICS_PORT` | `9091` | Port for the Prometheus metrics HTTP server. Metrics are available at `/metrics`. Set to `0` to disable. | +| `RECACHED_PORT` | `6379` | TCP port the RESP listener binds. Set it to run a second instance on one host — alongside a primary, for example — or to move off 6379, the first port a commodity scanner probes. The port is not a security control (`RECACHED_BIND`, `RECACHED_PASSWORD`, TLS and the allowlists are), so changing it hides nothing on its own. An invalid value, `0`, or a value equal to `RECACHED_WS_PORT` makes the server **refuse to start**, rather than falling back to 6379 and serving the keyspace on a port the operator believes is closed. Ports below 1024 require root on Unix. | +| `RECACHED_WS_PORT` | `6380` | TCP port the WebSocket sync listener binds. Same rules as `RECACHED_PORT`, and the two must differ. Running more than one instance per host means giving each its own `RECACHED_PORT`, `RECACHED_WS_PORT` and `RECACHED_METRICS_PORT`. | +| `RECACHED_METRICS_PORT` | `9091` | Port for the Prometheus metrics HTTP server. Metrics are available at `/metrics`. Set to `0` to disable the exporter entirely. An invalid value makes the server **refuse to start**. Before 0.2.5, `0` bound an OS-assigned ephemeral port instead of disabling anything, so metrics stayed exposed on an unpredictable port; and a collision on this port aborted startup with a panic rather than an explanation. | | `RECACHED_SAVE_PATH` | `recached.rdb` | Path to the snapshot file. The server loads this file on startup and writes to it on `SAVE`, `BGSAVE`, autosave, and clean shutdown. | | `RECACHED_SAVE` | _(none)_ | Multi-condition autosave policy as comma-separated `seconds:changes` pairs. A snapshot is triggered when **any** condition is satisfied: `elapsed_since_last_save >= seconds` **and** `dirty_writes >= changes`. Example: `"900:1,300:10,60:10000"` — save after 1 write in 15 min, 10 writes in 5 min, or 10 000 writes in 1 min. When set, `RECACHED_SAVE_INTERVAL` is ignored. Skips saves when no writes have occurred since the last snapshot. | | `RECACHED_SAVE_INTERVAL` | `900` | Autosave interval in seconds (single-condition fallback when `RECACHED_SAVE` is not set). The server saves automatically at this interval if at least one write has occurred since the last save. Set to `0` to disable autosave entirely (manual `SAVE`/`BGSAVE` still work). | diff --git a/sdks/recached-react/package.json b/sdks/recached-react/package.json index 166bb22..f03f823 100644 --- a/sdks/recached-react/package.json +++ b/sdks/recached-react/package.json @@ -1,6 +1,6 @@ { "name": "@recached/react", - "version": "0.2.4", + "version": "0.2.5", "description": "Official React hooks for Recached \u2014 zero-latency reactive cache", "type": "module", "main": "./dist/index.js", diff --git a/sdks/recached-vue/package.json b/sdks/recached-vue/package.json index 0b02eeb..7903b2a 100644 --- a/sdks/recached-vue/package.json +++ b/sdks/recached-vue/package.json @@ -1,6 +1,6 @@ { "name": "@recached/vue", - "version": "0.2.4", + "version": "0.2.5", "description": "Official Vue 3 composables for Recached \u2014 zero-latency reactive cache", "type": "module", "main": "./dist/index.js", diff --git a/server-native/Cargo.toml b/server-native/Cargo.toml index ff33d9f..e4e99b2 100644 --- a/server-native/Cargo.toml +++ b/server-native/Cargo.toml @@ -1,8 +1,21 @@ [package] -name = "server-native" +# The crate is `recached` and the binary it installs is `recached-server`, which +# is what the README, the Homebrew formula and the Docker image have always +# said. The package was `server-native` and produced a `server-native` binary, +# so `cargo install recached` installed nothing (the name was unregistered) and +# `cargo build --bin recached-server` failed outright; only Docker and brew +# worked, because both rename the artefact as they copy it. The directory keeps +# its name — it describes the role, the package describes the product. +name = "recached" version.workspace = true edition.workspace = true license.workspace = true +description = "A Rust cache server that runs on your backend and inside the browser." +repository.workspace = true + +[[bin]] +name = "recached-server" +path = "src/main.rs" [dependencies] core-engine.workspace = true diff --git a/server-native/src/clients.rs b/server-native/src/clients.rs new file mode 100644 index 0000000..c3ec2f1 --- /dev/null +++ b/server-native/src/clients.rs @@ -0,0 +1,372 @@ +//! Per-connection bookkeeping and command metrics: the client registry +//! CLIENT LIST reads, and the counters every executed command feeds. + +use crate::*; + +/// Counters mirrored out of the `metrics` registry so `INFO` can read them. +/// +/// `metrics::Counter` and `Gauge` handles are write-only — there is no way to +/// read a recorded value back out — so every number `INFO` reports from the +/// registry needs a plain atomic alongside it. These are the only ones INFO +/// needs; the rest of its fields come from the store or `ServerState`. +pub(crate) static STAT_CONNECTIONS_TOTAL: AtomicU64 = AtomicU64::new(0); + +pub(crate) static STAT_CONNECTIONS_ACTIVE: AtomicI64 = AtomicI64::new(0); + +pub(crate) static STAT_COMMANDS_TOTAL: AtomicU64 = AtomicU64::new(0); + +pub(crate) static STAT_KEYSPACE_HITS: AtomicU64 = AtomicU64::new(0); + +pub(crate) static STAT_KEYSPACE_MISSES: AtomicU64 = AtomicU64::new(0); + +/// RAII guard that tracks an active connection. Increments on creation, +/// decrements when dropped (i.e. when the handler future completes), and +/// keeps the `CLIENT LIST` registry in step with both. +pub(crate) struct ConnectionGuard { + pub(crate) id: u64, +} + +impl ConnectionGuard { + pub(crate) fn new(kind: &'static str, meta: ClientMeta) -> Self { + counter!("recached_connections_total", "type" => kind).increment(1); + gauge!("recached_connections_active").increment(1.0); + STAT_CONNECTIONS_TOTAL.fetch_add(1, Ordering::Relaxed); + STAT_CONNECTIONS_ACTIVE.fetch_add(1, Ordering::Relaxed); + let id = meta.id; + publish_client(meta); + Self { id } + } +} + +impl Drop for ConnectionGuard { + fn drop(&mut self) { + gauge!("recached_connections_active").decrement(1.0); + STAT_CONNECTIONS_ACTIVE.fetch_sub(1, Ordering::Relaxed); + CLIENTS + .write() + .unwrap_or_else(|e| e.into_inner()) + .remove(&self.id); + } +} + +// ── Client registry ─────────────────────────────────────────────────────────── + +/// What `CLIENT INFO` and `CLIENT LIST` report about one live connection. +/// +/// The connection task owns the authoritative copy and republishes it whenever +/// a field changes — a name is set, a library identifies itself, the protocol +/// is renegotiated. Publishing on change rather than per command keeps the +/// registry's write lock off the hot path: these events happen a handful of +/// times per connection, commands happen millions of times. +#[derive(Clone, Debug)] +pub(crate) struct ClientMeta { + pub(crate) id: u64, + /// Peer address, or empty when the listener could not report one. + pub(crate) addr: String, + pub(crate) laddr: String, + pub(crate) name: String, + pub(crate) lib_name: String, + pub(crate) lib_ver: String, + pub(crate) since: SystemTime, + pub(crate) resp: u8, + pub(crate) sub: usize, + pub(crate) psub: usize, +} + +impl ClientMeta { + pub(crate) fn new(id: u64, addr: String, laddr: String) -> Self { + Self { + id, + addr, + laddr, + name: String::new(), + lib_name: String::new(), + lib_ver: String::new(), + since: SystemTime::now(), + resp: 2, + sub: 0, + psub: 0, + } + } + + /// One line in Redis's `CLIENT LIST` format: space-separated `key=value`. + /// + /// Only fields Recached can answer truthfully are emitted. Redis also + /// reports buffer sizes, file descriptors and an event mask; inventing + /// plausible numbers for those would be worse than leaving them out, + /// because a client cannot tell a made-up `omem` from a real one. Parsers + /// read this format key by key and skip what they do not recognise, so a + /// shorter line is a supported line. + pub(crate) fn render(&self) -> String { + let age = self.since.elapsed().unwrap_or_default().as_secs(); + format!( + "id={} addr={} laddr={} name={} age={} idle=0 flags=N db=0 \ + sub={} psub={} multi=-1 resp={} lib-name={} lib-ver={}", + self.id, + self.addr, + self.laddr, + self.name, + age, + self.sub, + self.psub, + self.resp, + self.lib_name, + self.lib_ver, + ) + } +} + +/// Live connections, keyed by id. A `BTreeMap` so `CLIENT LIST` comes out in +/// connection order rather than an arbitrary one that shuffles between calls. +pub(crate) static CLIENTS: std::sync::LazyLock>> = + std::sync::LazyLock::new(Default::default); + +pub(crate) fn publish_client(meta: ClientMeta) { + CLIENTS + .write() + .unwrap_or_else(|e| e.into_inner()) + .insert(meta.id, meta); +} + +pub(crate) fn client_list_lines() -> String { + let clients = CLIENTS.read().unwrap_or_else(|e| e.into_inner()); + let mut out = String::new(); + for meta in clients.values() { + out.push_str(&meta.render()); + out.push('\n'); + } + out +} + +pub(crate) fn command_name(cmd: &Command) -> &'static str { + match cmd { + Command::Ping(_) => "ping", + Command::Auth(_) => "auth", + Command::Hello(_) => "hello", + Command::Info(_) => "info", + Command::Quit => "quit", + Command::Client(_) => "client", + Command::Config(_) => "config", + Command::CommandQuery(_) => "command", + Command::Cluster(_) => "cluster", + Command::Module(_) => "module", + Command::PubSub(_) => "pubsub", + Command::Memory(_) | Command::MemoryUsage(_) => "memory", + Command::Get(_) => "get", + Command::ESet(_, _) => "eset", + Command::Set(_, _, _) => "set", + Command::Del(_) => "del", + Command::Unlink(_) => "unlink", + Command::Append(_, _) => "append", + Command::Strlen(_) => "strlen", + Command::GetRange(_, _, _) => "getrange", + Command::GetSet(_, _) => "getset", + Command::MGet(_) => "mget", + Command::MSet(_) => "mset", + Command::SetNx(_, _) => "setnx", + Command::SetEx(_, _, _) => "setex", + Command::PSetEx(_, _, _) => "psetex", + Command::Incr(_) => "incr", + Command::Decr(_) => "decr", + Command::IncrBy(_, _) => "incrby", + Command::DecrBy(_, _) => "decrby", + Command::Expire(_, _) => "expire", + Command::PExpire(_, _) => "pexpire", + Command::ExpireAt(_, _) => "expireat", + Command::PExpireAt(_, _) => "pexpireat", + Command::Ttl(_) => "ttl", + Command::PTtl(_) => "pttl", + Command::Persist(_) => "persist", + Command::Exists(_) => "exists", + Command::Keys(_) => "keys", + Command::Scan(_, _, _) => "scan", + Command::DbSize => "dbsize", + Command::FlushDb => "flushdb", + Command::Rename(_, _) => "rename", + Command::Type(_) => "type", + Command::HSet(_, _) => "hset", + Command::HGet(_, _) => "hget", + Command::HGetAll(_) => "hgetall", + Command::HDel(_, _) => "hdel", + Command::HKeys(_) => "hkeys", + Command::HVals(_) => "hvals", + Command::HLen(_) => "hlen", + Command::HIncrBy(_, _, _) => "hincrby", + Command::HIncrByFloat(_, _, _) => "hincrbyfloat", + Command::HExists(_, _) => "hexists", + Command::HSetNx(_, _, _) => "hsetnx", + Command::HMGet(_, _) => "hmget", + Command::HScan(_, _) => "hscan", + Command::LPush(_, _) => "lpush", + Command::RPush(_, _) => "rpush", + Command::LPushX(_, _) => "lpushx", + Command::RPushX(_, _) => "rpushx", + Command::LPop(_, _) => "lpop", + Command::RPop(_, _) => "rpop", + Command::LRange(_, _, _) => "lrange", + Command::LLen(_) => "llen", + Command::LIndex(_, _) => "lindex", + Command::LSet(_, _, _) => "lset", + Command::LRem(_, _, _) => "lrem", + Command::LTrim(_, _, _) => "ltrim", + Command::SAdd(_, _) => "sadd", + Command::SMembers(_) => "smembers", + Command::SRem(_, _) => "srem", + Command::SCard(_) => "scard", + Command::SIsMember(_, _) => "sismember", + Command::SMIsMember(_, _) => "smismember", + Command::SInter(_) => "sinter", + Command::SInterStore(_, _) => "sinterstore", + Command::SUnion(_) => "sunion", + Command::SUnionStore(_, _) => "sunionstore", + Command::SDiff(_) => "sdiff", + Command::SDiffStore(_, _) => "sdiffstore", + Command::SPop(_, _) => "spop", + Command::SRandMember(_, _) => "srandmember", + Command::SMove(_, _, _) => "smove", + Command::SScan(_, _) => "sscan", + Command::ZAdd(_, _, _) => "zadd", + Command::ZRange(_, _, _, _) => "zrange", + Command::ZRevRange(_, _, _, _) => "zrevrange", + Command::ZRangeByScore(_, _, _, _, _) => "zrangebyscore", + Command::ZRevRangeByScore(_, _, _, _, _) => "zrevrangebyscore", + Command::ZScore(_, _) => "zscore", + Command::ZMScore(_, _) => "zmscore", + Command::ZRank(_, _) => "zrank", + Command::ZRevRank(_, _) => "zrevrank", + Command::ZRem(_, _) => "zrem", + Command::ZCard(_) => "zcard", + Command::ZIncrBy(_, _, _) => "zincrby", + Command::ZCount(_, _, _) => "zcount", + Command::ZScan(_, _) => "zscan", + Command::Multi => "multi", + Command::Exec => "exec", + Command::Discard => "discard", + Command::Subscribe(_) => "subscribe", + Command::Unsubscribe(_) => "unsubscribe", + Command::PSubscribe(_) => "psubscribe", + Command::PUnsubscribe(_) => "punsubscribe", + Command::Publish(_, _) => "publish", + Command::Watch(_) => "watch", + Command::Unwatch(_) => "unwatch", + Command::Save => "save", + Command::BgSave => "bgsave", + Command::LastSave => "lastsave", + Command::ReplicaOfNoOne => "replicaof", + Command::JSet(_, _, _) => "jset", + Command::JGet(_, _) => "jget", + Command::JMerge(_, _) => "jmerge", + Command::RlSet(_, _, _) => "rlset", + Command::RlCheck(_, _) => "rlcheck", + Command::Sync(_) => "sync", + Command::QSub(_) => "qsub", + Command::QUnsub(_) => "qunsub", + // Metrics count the wrapped command, not the envelope. + Command::Dedup(_, _, inner) => command_name(inner), + Command::Unknown(_) => UNKNOWN_COMMAND, + } +} + +/// Per-command counter handles, resolved through the metrics registry once and +/// then reused — the registry lookup (key construction + shard lock) is too +/// expensive to pay on every command. Keyed by the `&'static str` from +/// `command_name`. +/// +/// Built in one shot from the command catalog on first use, which is after +/// `main` has installed the recorder, and never written again. It used to be an +/// `RwLock` filled in as each command was first seen, which cost a +/// read-lock acquisition on *every command on every connection* — one +/// contended atomic in the hot path of a server whose whole job is throughput — +/// and `.unwrap()`ed the lock, so a single panic anywhere holding it would +/// poison the lock and make every subsequent command panic for the life of the +/// process. An immutable map needs no lock and cannot be poisoned. +pub(crate) static CMD_COUNTERS: std::sync::LazyLock> = + std::sync::LazyLock::new(|| { + core_engine::catalog::CATALOG + .iter() + .map(|spec| { + ( + spec.name, + counter!("recached_commands_total", "command" => spec.name), + ) + }) + .chain(std::iter::once(( + UNKNOWN_COMMAND, + counter!("recached_commands_total", "command" => UNKNOWN_COMMAND), + ))) + .collect() + }); + +/// Label used for a command the parser did not recognise. Not a catalog row, so +/// it is registered alongside them. +pub(crate) const UNKNOWN_COMMAND: &str = "unknown"; + +pub(crate) fn record_command(name: &'static str) { + STAT_COMMANDS_TOTAL.fetch_add(1, Ordering::Relaxed); + match CMD_COUNTERS.get(name) { + Some(c) => c.increment(1), + // A name `command_name` can produce but the catalog does not list. + // `command_name_labels_are_all_pre_registered` fails CI if that ever + // happens, so this is a correctness backstop, not a routine path. + None => counter!("recached_commands_total", "command" => name).increment(1), + } +} + +pub(crate) static KEYSPACE_HITS: std::sync::LazyLock = + std::sync::LazyLock::new(|| counter!("recached_keyspace_hits_total")); + +pub(crate) static KEYSPACE_MISSES: std::sync::LazyLock = + std::sync::LazyLock::new(|| counter!("recached_keyspace_misses_total")); + +/// Executes `cmd`, recording metrics and the dirty counter. Takes the command +/// by value — the hot path hands it straight to the store without a clone; +/// callers that still need the command afterwards (write fan-out) clone first. +pub(crate) fn execute_and_record(store: &KeyValueStore, cmd: Command) -> Value { + let name = command_name(&cmd); + let is_write = is_write_command(&cmd); + let is_get = matches!(cmd, Command::Get(_)); + let response = store.execute(cmd); + record_command(name); + if matches!(response, Value::Error(_)) { + counter!("recached_command_errors_total", "command" => name).increment(1); + } else if is_write { + store.mark_dirty(); + } + if is_get { + match &response { + Value::BulkString(Some(_)) => { + KEYSPACE_HITS.increment(1); + STAT_KEYSPACE_HITS.fetch_add(1, Ordering::Relaxed); + } + Value::BulkString(None) => { + KEYSPACE_MISSES.increment(1); + STAT_KEYSPACE_MISSES.fetch_add(1, Ordering::Relaxed); + } + _ => {} + } + } + response +} + +/// True when at least one consumer of write effects exists (WebSocket peers, +/// AOF, replicas, or watched keys). When false — the common standalone case — +/// the caller can move the command into `execute_and_record` without cloning +/// and skip `apply_write_effects` entirely. +pub(crate) fn write_effects_armed( + tx: &broadcast::Sender, + state: &ServerState, + watch_registry: &WatchRegistry, +) -> bool { + tx.receiver_count() > 0 || state.needs_write_log() || !watch_registry.is_empty() +} + +// ── TCP listeners ───────────────────────────────────────────────────────────── + +// TCP mutation broadcasts use id=0; WS/TCP pubsub connections get ids ≥ 1. +pub(crate) static NEXT_CONN_ID: AtomicU64 = AtomicU64::new(1); + +pub(crate) fn next_conn_id() -> u64 { + NEXT_CONN_ID.fetch_add(1, Ordering::Relaxed) +} + +// ── pub/sub ─────────────────────────────────────────────────────────────────── diff --git a/server-native/src/config.rs b/server-native/src/config.rs new file mode 100644 index 0000000..f2bff84 --- /dev/null +++ b/server-native/src/config.rs @@ -0,0 +1,600 @@ +//! Environment-driven configuration: limits, ports, allowlists and the +//! parsing rules that decide whether a value is honoured or refuses startup. + +use crate::*; + +pub(crate) const TCP_READ_BUFFER_BYTES: usize = 16 * 1024; // 16 KB — matches Redis default + +pub(crate) const MAX_TCP_READ_BUFFER_BYTES: usize = 64 * 1024 * 1024; // 64 MB per connection + +/// Per-connection limits. Compiled-in defaults, overridable at startup because +/// the right value is workload-dependent — Redis exposes `maxmemory-samples` +/// for the same reason. Read once and cached; changing one needs a restart. +pub(crate) fn env_limit(var: &str, default: usize) -> usize { + std::env::var(var) + .ok() + .and_then(|v| v.trim().parse::().ok()) + .filter(|n| *n > 0) + .unwrap_or(default) +} + +/// Commands queued inside one `MULTI`. Override: `RECACHED_MAX_MULTI_QUEUE`. +pub(crate) fn max_multi_queue_len() -> usize { + static V: std::sync::OnceLock = std::sync::OnceLock::new(); + *V.get_or_init(|| env_limit("RECACHED_MAX_MULTI_QUEUE", 10_000)) +} + +/// Keys one connection may `WATCH`. Override: `RECACHED_MAX_WATCHES_PER_CONN`. +pub(crate) fn max_watches_per_conn() -> usize { + static V: std::sync::OnceLock = std::sync::OnceLock::new(); + *V.get_or_init(|| env_limit("RECACHED_MAX_WATCHES_PER_CONN", 1_024)) +} + +/// Live queries one connection may hold. Override: `RECACHED_MAX_LIVE_QUERIES`. +pub(crate) fn max_qsubs_per_conn() -> usize { + static V: std::sync::OnceLock = std::sync::OnceLock::new(); + *V.get_or_init(|| env_limit("RECACHED_MAX_LIVE_QUERIES", 64)) +} + +/// Cap on the number of key/value pairs returned as QSUB initial state, so a +/// pattern matching a huge keyspace cannot produce an unbounded reply frame. +/// Keys returned in a live query's initial state. +/// Override: `RECACHED_MAX_QSUB_INITIAL_KEYS`. +pub(crate) fn max_qsub_initial_keys() -> usize { + static V: std::sync::OnceLock = std::sync::OnceLock::new(); + *V.get_or_init(|| env_limit("RECACHED_MAX_QSUB_INITIAL_KEYS", 10_000)) +} + +pub(crate) const BROADCAST_CHANNEL_CAPACITY: usize = 512; + +pub(crate) const DEFAULT_MAX_CONNECTIONS: usize = 1024; + +pub(crate) const MAX_AUTH_FAILURES: u32 = 5; + +pub(crate) const EVICTION_INTERVAL_SECS: u64 = 1; + +pub(crate) const DEFAULT_HANDSHAKE_TIMEOUT_SECS: u64 = 10; + +/// Longest replication auth line accepted, in bytes. +pub(crate) const MAX_REPL_AUTH_LINE: usize = 512; + +/// Window over which failed replication auth attempts are counted per peer. +pub(crate) const REPL_AUTH_WINDOW: Duration = Duration::from_secs(60); + +/// Peers tracked before the throttle sweeps expired entries. +pub(crate) const REPL_AUTH_SWEEP_THRESHOLD: usize = 1024; + +// ── private file writes ─────────────────────────────────────────────────────── + +/// Parse a human-readable memory size string (e.g. "512mb", "1gb", "262144") +/// into a byte count. Returns None on parse failure. +/// Parse `RECACHED_ALLOW_IPS` into exact addresses. +/// +/// Every entry must parse. The previous behaviour logged and dropped invalid +/// entries, which quietly narrowed a security control: a mistyped CIDR range +/// like `10.0.0.0/8` produced an allowlist that did not include the hosts the +/// operator wrote, and an entirely invalid list produced an empty one — which +/// rejects *every* connection while the server still reports itself healthy. +/// Refusing to start makes the misconfiguration impossible to miss. +pub(crate) fn parse_allow_ips(raw: &str) -> Result, String> { + let mut ips = Vec::new(); + for entry in raw.split(',') { + let trimmed = entry.trim(); + if trimmed.is_empty() { + continue; + } + match IpAddr::from_str(trimmed) { + Ok(ip) => ips.push(ip), + Err(_) => { + return Err(format!( + "RECACHED_ALLOW_IPS: '{trimmed}' is not a valid IP address. Exact addresses \ + only — CIDR ranges and hostnames are not supported. Refusing to start rather \ + than applying a narrower allowlist than configured." + )); + } + } + } + if ips.is_empty() { + return Err( + "RECACHED_ALLOW_IPS is set but contains no valid addresses — this would reject every \ + connection. Unset it to accept all connections." + .to_string(), + ); + } + Ok(ips) +} + +/// Parse a boolean environment variable, rejecting anything ambiguous. +/// +/// Silently treating `RECACHED_REPL_ENABLE=please` as false would leave an +/// operator believing replication was on when it was not; treating it as true +/// would open a port they never asked for. Neither is acceptable for a variable +/// that gates a security boundary, so an unrecognised value refuses to start. +pub(crate) fn parse_env_bool(var: &str, raw: &str) -> Result { + match raw.trim().to_ascii_lowercase().as_str() { + "1" | "true" | "yes" | "on" => Ok(true), + "0" | "false" | "no" | "off" => Ok(false), + other => Err(format!( + "{var}: '{other}' is not a boolean. Use 1/true/yes/on or 0/false/no/off." + )), + } +} + +/// Resolve a listening port from the environment, or `default` when unset. +/// +/// A typo refuses to start rather than falling back. Silently serving 6379 to +/// an operator who asked for 7000 puts the keyspace on a port they believe is +/// closed — the same reasoning as [`parse_env_bool`]. Port 0 is rejected for +/// the same reason: the OS would assign an arbitrary free port and nothing +/// would be reachable at the address anyone was told to use. +pub(crate) fn parse_env_port(var: &str, default: u16) -> Result { + match std::env::var(var) { + Err(_) => Ok(default), + Ok(raw) => match raw.trim().parse::() { + Ok(0) | Err(_) => Err(format!( + "{var}: '{}' is not a valid port. Use 1-65535.", + raw.trim() + )), + Ok(p) => Ok(p), + }, + } +} + +/// The metrics port, or `None` when metrics are switched off. +/// +/// Unlike the data ports, `0` is meaningful here — the documented way to turn +/// the exporter off — so it is answered rather than refused. Everything else is +/// validated the same way, because a typo silently exporting on 9091 is the +/// same failure as a typo silently serving the keyspace on 6379. +pub(crate) fn parse_env_metrics_port() -> Result, String> { + const VAR: &str = "RECACHED_METRICS_PORT"; + match std::env::var(VAR) { + Err(_) => Ok(Some(9091)), + Ok(raw) => match raw.trim().parse::() { + Ok(0) => Ok(None), + Ok(p) => Ok(Some(p)), + Err(_) => Err(format!( + "{VAR}: '{}' is not a valid port. Use 1-65535, or 0 to disable metrics.", + raw.trim() + )), + }, + } +} + +/// True when `bind_host` can only be reached from this machine. +/// +/// A hostname that does not parse as an address is treated as public: the +/// conservative answer is the one that demands a password. +pub(crate) fn bind_is_loopback(bind_host: &str) -> bool { + if bind_host.eq_ignore_ascii_case("localhost") { + return true; + } + // An IPv6 bind address has to be written bracketed — `[::1]` — because the + // listeners format it as `{host}:{port}`, and `::1:6379` does not parse. + // Without stripping them, `[::1]` fails to parse as an address and would be + // classified as public, demanding a replication password on what is in fact + // loopback. + let host = bind_host + .strip_prefix('[') + .and_then(|h| h.strip_suffix(']')) + .unwrap_or(bind_host); + IpAddr::from_str(host) + .map(|ip| ip.is_loopback()) + .unwrap_or(false) +} + +/// Whether to bind the replication listener, given the environment. +/// +/// This listener used to start unconditionally on every node. The stated reason +/// was multi-tier replication — a replica must be able to serve sub-replicas — +/// but the cost was that *every* default deployment opened a port carrying the +/// entire keyspace to anyone who connected: with no `RECACHED_REPL_PASSWORD`, +/// `handle_replica` skips the handshake and sends a full snapshot followed by a +/// live stream of every subsequent write. An operator who set +/// `RECACHED_PASSWORD` had every reason to believe the data was behind +/// authentication, and it was not. +/// +/// Two rules now apply. The port is closed unless `RECACHED_REPL_ENABLE` says +/// otherwise, and enabling it on an interface other than loopback without a +/// password refuses to start rather than serving the keyspace unauthenticated. +/// Multi-tier replication still works — a node that serves sub-replicas sets +/// the variable, which is the point: it is now a decision rather than a default. +pub(crate) fn resolve_repl_listen( + enable: Option, + bind_host: &str, + password: Option<&str>, +) -> Result { + let enabled = match enable.as_deref().map(str::trim) { + None | Some("") => false, + Some(v) => parse_env_bool("RECACHED_REPL_ENABLE", v)?, + }; + if !enabled { + return Ok(false); + } + let has_password = password.is_some_and(|p| !p.is_empty()); + if !has_password && !bind_is_loopback(bind_host) { + return Err(format!( + "RECACHED_REPL_ENABLE is set and RECACHED_BIND is '{bind_host}', but \ + RECACHED_REPL_PASSWORD is unset — refusing to start. The replication port serves the \ + entire keyspace to whoever connects, so on any interface reachable from the network \ + it must be authenticated. Set RECACHED_REPL_PASSWORD, or bind to 127.0.0.1." + )); + } + Ok(true) +} + +/// Parse `RECACHED_ALLOWED_ORIGINS` into exact origins. +/// +/// An origin is scheme + host + optional port and nothing else, so an entry +/// carrying a path is a misunderstanding of what will be compared against — and +/// one that would silently never match. Reject it at startup, in the same +/// spirit as `parse_allow_ips`. +pub(crate) fn parse_allowed_origins(raw: &str) -> Result, String> { + let mut origins = Vec::new(); + for entry in raw.split(',') { + let trimmed = entry.trim().trim_end_matches('/'); + if trimmed.is_empty() { + continue; + } + // Sandboxed iframes and `file://` documents send the literal `null`. + // An operator may legitimately need to admit them. + if trimmed.eq_ignore_ascii_case("null") { + origins.push("null".to_string()); + continue; + } + let Some((scheme, authority)) = trimmed.split_once("://") else { + return Err(format!( + "RECACHED_ALLOWED_ORIGINS: '{trimmed}' is not an origin — it needs a scheme, e.g. \ + https://app.example.com." + )); + }; + if scheme.is_empty() || authority.is_empty() { + return Err(format!( + "RECACHED_ALLOWED_ORIGINS: '{trimmed}' is not an origin — expected \ + scheme://host[:port]." + )); + } + if authority.contains('/') { + return Err(format!( + "RECACHED_ALLOWED_ORIGINS: '{trimmed}' contains a path. An origin is \ + scheme://host[:port] only, and a browser will never send a path — this entry \ + could never match." + )); + } + origins.push(trimmed.to_ascii_lowercase()); + } + if origins.is_empty() { + return Err( + "RECACHED_ALLOWED_ORIGINS is set but lists no origins — this would reject every \ + browser. Unset it to accept all origins." + .to_string(), + ); + } + Ok(origins) +} + +/// Whether a WebSocket handshake carrying `origin` may proceed. +/// +/// Browsers apply neither CORS nor a preflight to WebSockets, so without this +/// check any page a user visits can open a socket to a reachable Recached and +/// act with that user's network position. On the common `ws://localhost:6380` +/// development setup that means every site in every tab. +/// +/// `Origin` is not a boundary against a native client, which simply omits the +/// header — and that is why an absent origin is allowed. What it does +/// distinguish is "the application I deployed" from "some other page in the same +/// browser", which is precisely the threat this port faces. An unset allowlist +/// permits everything, matching how an unset `RECACHED_PASSWORD` behaves. +pub(crate) fn origin_allowed(allowed: Option<&[String]>, origin: Option<&str>) -> bool { + let Some(list) = allowed else { + return true; + }; + let Some(origin) = origin else { + return true; + }; + let origin = origin.trim().trim_end_matches('/'); + list.iter().any(|a| a.eq_ignore_ascii_case(origin)) +} + +/// How long a connection may take to complete its TLS and/or WebSocket +/// handshake. Override: `RECACHED_HANDSHAKE_TIMEOUT` (seconds). +/// +/// The connection permit is taken before the handshake runs, so without a +/// deadline a client that opens a socket and then says nothing holds one of +/// `RECACHED_MAX_CONNECTIONS` slots indefinitely. A thousand such sockets cost +/// an attacker nothing and stop the server accepting real clients. +pub(crate) fn handshake_timeout() -> Duration { + static V: std::sync::OnceLock = std::sync::OnceLock::new(); + Duration::from_secs(*V.get_or_init(|| { + env_limit( + "RECACHED_HANDSHAKE_TIMEOUT", + DEFAULT_HANDSHAKE_TIMEOUT_SECS as usize, + ) as u64 + })) +} + +pub(crate) fn parse_memory_bytes(s: &str) -> Option { + let s = s.trim().to_lowercase(); + if let Some(n) = s.strip_suffix("gb") { + n.trim() + .parse::() + .ok() + .map(|n| n * 1024 * 1024 * 1024) + } else if let Some(n) = s.strip_suffix("mb") { + n.trim().parse::().ok().map(|n| n * 1024 * 1024) + } else if let Some(n) = s.strip_suffix("kb") { + n.trim().parse::().ok().map(|n| n * 1024) + } else { + s.parse().ok() + } +} + +/// A single autosave condition: save if `changes` or more writes have +/// accumulated within `secs` seconds of the last save. +pub(crate) struct SaveCondition { + pub(crate) secs: u64, + pub(crate) changes: u64, +} + +/// Parse `RECACHED_SAVE` value: comma-separated `seconds:changes` pairs. +/// Example: `"900:1,300:10,60:10000"` → save after 1 change in 15 min, +/// 10 changes in 5 min, or 10 000 changes in 1 min — whichever comes first. +pub(crate) fn parse_save_conditions(s: &str) -> Vec { + s.split(',') + .filter_map(|pair| { + let mut parts = pair.trim().splitn(2, ':'); + let secs: u64 = parts.next()?.trim().parse().ok()?; + let changes: u64 = parts.next()?.trim().parse().ok()?; + Some(SaveCondition { secs, changes }) + }) + .collect() +} + +// ── Replication server (primary side) ──────────────────────────────────────── + +#[cfg(test)] +mod limit_config_tests { + use super::*; + + /// Serialises the tests in this module. + /// + /// Environment variables are process-global and `cargo test` runs tests on + /// parallel threads, so a test that sets `RECACHED_MAX_*` races any test + /// reading the same variable — which is why `set_var` is `unsafe`. This + /// surfaced as `compiled_defaults_match_the_documented_values` + /// intermittently observing an override (`7`) instead of a default + /// (`10_000`): it passed locally and failed in CI purely on thread timing. + /// + /// Poisoning is ignored deliberately: one failing test must not cascade + /// into unrelated failures in the rest of the module. + static ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + + fn env_guard() -> std::sync::MutexGuard<'static, ()> { + ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()) + } + + #[test] + fn env_limit_falls_back_to_the_default() { + let _guard = env_guard(); + // Unset, empty, non-numeric, and zero all mean "use the default" — + // a zero limit would disable the feature rather than tune it. + assert_eq!(env_limit("RECACHED_DEFINITELY_UNSET_VAR_XYZ", 64), 64); + for bad in ["", " ", "abc", "0", "-5", "1.5"] { + unsafe { std::env::set_var("RECACHED_TEST_LIMIT", bad) }; + assert_eq!(env_limit("RECACHED_TEST_LIMIT", 64), 64, "input {bad:?}"); + } + unsafe { std::env::remove_var("RECACHED_TEST_LIMIT") }; + } + + #[test] + fn env_limit_accepts_a_positive_override() { + let _guard = env_guard(); + unsafe { std::env::set_var("RECACHED_TEST_LIMIT_OK", " 256 ") }; + assert_eq!( + env_limit("RECACHED_TEST_LIMIT_OK", 64), + 256, + "whitespace tolerated" + ); + unsafe { std::env::remove_var("RECACHED_TEST_LIMIT_OK") }; + } + + #[test] + fn overrides_are_read_from_the_documented_variable_names() { + let _guard = env_guard(); + // A bulk rename once rewrote these string literals along with the + // function names, leaving variables like `RECACHED_max_watches_per_conn()` + // that no operator would ever set — the override silently did nothing. + // Assert the names the docs promise. + for (var, default) in [ + ("RECACHED_MAX_MULTI_QUEUE", 10_000usize), + ("RECACHED_MAX_WATCHES_PER_CONN", 1_024), + ("RECACHED_MAX_LIVE_QUERIES", 64), + ("RECACHED_MAX_QSUB_INITIAL_KEYS", 10_000), + ("RECACHED_EVICTION_SAMPLE", 10), + ] { + assert!( + var.chars() + .all(|c| c.is_ascii_uppercase() || c == '_' || c.is_ascii_digit()), + "{var} is not a plausible environment variable name" + ); + unsafe { std::env::set_var(var, "7") }; + assert_eq!(env_limit(var, default), 7, "{var} override ignored"); + unsafe { std::env::remove_var(var) }; + } + } + + #[test] + fn compiled_defaults_match_the_documented_values() { + let _guard = env_guard(); + // These appear in docs/server/operations.md; drift would mislead + // operators sizing a deployment. + assert_eq!(max_multi_queue_len(), 10_000); + assert_eq!(max_watches_per_conn(), 1_024); + assert_eq!(max_qsubs_per_conn(), 64); + assert_eq!(max_qsub_initial_keys(), 10_000); + } +} + +/// The TCP and WebSocket ports were compiled in, so two instances could not +/// share a host — a replica beside its primary was impossible — and there was +/// no way off 6379, the first port any commodity scanner probes. +/// +/// The port is not itself a security control (`RECACHED_BIND`, the password, +/// TLS and the allowlists are), so it is safe to expose. What is *not* safe is +/// a typo falling back to the default: an operator who asked for 7000 and +/// silently got 6379 believes a port is closed that is in fact serving the +/// keyspace. So a bad value refuses to start. +#[cfg(test)] +mod port_config_tests { + use super::*; + + /// `std::env` is process-global; serialise the tests that mutate it. + static ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + + fn with_var(key: &str, value: Option<&str>, f: impl FnOnce() -> T) -> T { + let _guard = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + let prev = std::env::var(key).ok(); + unsafe { + match value { + Some(v) => std::env::set_var(key, v), + None => std::env::remove_var(key), + } + } + let out = f(); + unsafe { + match prev { + Some(p) => std::env::set_var(key, p), + None => std::env::remove_var(key), + } + } + out + } + + const VAR: &str = "RECACHED_TEST_PORT_VAR"; + + #[test] + fn an_unset_variable_keeps_the_default() { + with_var(VAR, None, || { + assert_eq!(parse_env_port(VAR, 6379), Ok(6379)); + }); + } + + #[test] + fn a_valid_port_is_taken_verbatim() { + for (raw, want) in [ + ("7000", 7000u16), + ("1", 1), + ("65535", 65535), + (" 6380 ", 6380), + ] { + with_var(VAR, Some(raw), || { + assert_eq!(parse_env_port(VAR, 6379), Ok(want), "raw {raw:?}"); + }); + } + } + + #[test] + fn a_typo_refuses_to_start_rather_than_serving_the_default() { + // The whole point: silently falling back would put the keyspace on a + // port the operator believes is closed. + for raw in ["seven thousand", "63.79", "-1", "65536", "99999", ""] { + with_var(VAR, Some(raw), || { + let err = parse_env_port(VAR, 6379).unwrap_err_or_else_msg(raw); + assert!(err.contains(VAR), "{raw:?} -> {err}"); + assert!(err.contains("not a valid port"), "{raw:?} -> {err}"); + }); + } + } + + #[test] + fn port_zero_is_refused() { + // The OS would assign an arbitrary free port, so nothing would be + // reachable at the address anyone was told to use. + with_var(VAR, Some("0"), || { + assert!(parse_env_port(VAR, 6379).is_err()); + }); + } + + /// Small helper so the loop above reads as one assertion per case. + trait UnwrapErrMsg { + fn unwrap_err_or_else_msg(self, raw: &str) -> String; + } + impl UnwrapErrMsg for Result { + fn unwrap_err_or_else_msg(self, raw: &str) -> String { + match self { + Ok(p) => panic!("{raw:?} should have been refused, got port {p}"), + Err(e) => e, + } + } + } +} + +// ── PUBLISH inside MULTI ────────────────────────────────────────────────────── + +/// `RECACHED_METRICS_PORT=0` disables the exporter. +/// +/// The reference has always documented this; the code never honoured it. `0` +/// parsed fine and `bind("host:0")` hands the listener an OS-assigned ephemeral +/// port, so an operator switching metrics *off* got them served on an +/// unpredictable port instead — the opposite of the request, and unlikely to be +/// noticed until something scraped it. +#[cfg(test)] +mod metrics_port_tests { + use super::*; + + static ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + const VAR: &str = "RECACHED_METRICS_PORT"; + + fn with_metrics_port(value: Option<&str>, f: impl FnOnce() -> T) -> T { + let _guard = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()); + let prev = std::env::var(VAR).ok(); + unsafe { + match value { + Some(v) => std::env::set_var(VAR, v), + None => std::env::remove_var(VAR), + } + } + let out = f(); + unsafe { + match prev { + Some(p) => std::env::set_var(VAR, p), + None => std::env::remove_var(VAR), + } + } + out + } + + #[test] + fn zero_means_disabled_not_an_ephemeral_port() { + with_metrics_port(Some("0"), || { + assert_eq!( + parse_env_metrics_port(), + Ok(None), + "0 must switch the exporter off, not bind an OS-assigned port" + ); + }); + } + + #[test] + fn unset_keeps_the_default_port() { + with_metrics_port(None, || { + assert_eq!(parse_env_metrics_port(), Ok(Some(9091))); + }); + } + + #[test] + fn a_real_port_is_taken_verbatim() { + with_metrics_port(Some("9092"), || { + assert_eq!(parse_env_metrics_port(), Ok(Some(9092))); + }); + } + + #[test] + fn a_typo_refuses_to_start_and_mentions_the_disable_value() { + with_metrics_port(Some("nine thousand"), || { + let err = parse_env_metrics_port().unwrap_err(); + assert!(err.contains(VAR), "{err}"); + assert!(err.contains("0 to disable"), "{err}"); + }); + } +} diff --git a/server-native/src/connection.rs b/server-native/src/connection.rs new file mode 100644 index 0000000..fc9fc23 --- /dev/null +++ b/server-native/src/connection.rs @@ -0,0 +1,1528 @@ +//! Connection handling: the RESP and WebSocket command loops, the listeners +//! they run on, and the HELLO/AUTH handshakes that precede them. + +use crate::*; + +/// Binds `n` TCP sockets on `addr`, all with `SO_REUSEPORT`, so the OS can +/// distribute incoming connections across multiple accept loops — one per +/// Tokio worker thread. Falls back to a single plain `TcpListener::bind` on +/// platforms that don't support `SO_REUSEPORT`. +pub(crate) fn make_tcp_listeners(addr: &str, n: usize) -> std::io::Result> { + use socket2::{Domain, Socket, Type}; + let socket_addr: std::net::SocketAddr = addr + .parse() + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidInput, e))?; + let domain = if socket_addr.is_ipv6() { + Domain::IPV6 + } else { + Domain::IPV4 + }; + // SO_REUSEPORT — which lets multiple sockets share one port — is Unix-only. + // Without it, binding a second socket to the same port fails, so fall back + // to a single accept loop on non-Unix platforms. + #[cfg(not(unix))] + let n = 1; + let mut out = Vec::with_capacity(n); + for _ in 0..n { + let sock = Socket::new(domain, Type::STREAM, None)?; + sock.set_reuse_address(true)?; + #[cfg(unix)] + sock.set_reuse_port(true)?; + sock.set_nonblocking(true)?; + sock.bind(&socket_addr.into())?; + sock.listen(4096)?; + let std_listener: std::net::TcpListener = sock.into(); + out.push(TcpListener::from_std(std_listener)?); + } + Ok(out) +} + +// ── TLS ─────────────────────────────────────────────────────────────────────── + +/// `EXEC`'s reply when any command failed to queue. Byte-identical to Redis's, +/// because clients match on the `EXECABORT` prefix to tell "the transaction was +/// refused" from "a command inside it returned an error". +pub(crate) const EXECABORT: &[u8] = + b"-EXECABORT Transaction discarded because of previous errors.\r\n"; + +/// The reply for a command that must be *refused* at queue time inside `MULTI`, +/// or `None` if it may be queued. +/// +/// Redis rejects an unknown verb when it is queued, not when the transaction +/// runs, and that difference is the whole point: a queue-time rejection also +/// poisons the transaction so `EXEC` runs nothing. Recached parses an +/// unrecognised verb into [`Command::Unknown`], which used to queue happily and +/// only error during `EXEC` — leaving every *other* command in the transaction +/// applied. Against a server that deliberately implements a subset of Redis, +/// that is a live hazard: `MULTI; ZPOPMIN q; LPUSH processing x; EXEC` pushed +/// onto `processing` without ever popping `q`, silently, and MULTI is exactly +/// the construct callers reach for to prevent that. +/// +/// The wording matches what the store returns for the same command outside a +/// transaction, so a client sees one message for one mistake. +pub(crate) fn queue_time_rejection(cmd: &Command) -> Option> { + match cmd { + Command::Unknown(name) => { + Some(Value::Error(format!("ERR unknown command '{}'", name)).serialize()) + } + _ => None, + } +} + +/// Encode a pub/sub delivery for a connection speaking protocol `protover`. +/// +/// RESP2 has no push type, so a subscribed RESP2 client expects a plain array +/// and cannot parse a `>` frame at all. RESP3 clients want the push type so +/// deliveries are distinguishable from command replies on a multiplexed +/// connection. The WebSocket transport is RESP3 by definition — the sync +/// protocol is specified in terms of push frames — and passes 3. +/// Handle `HELLO [protover]`, updating `protover` in place on success. +/// +/// Returns the serialized reply. An unsupported version leaves the connection's +/// current protocol untouched and replies `-NOPROTO`, which is what lets a +/// client probe for RESP3 and fall back cleanly rather than being disconnected. +pub(crate) fn process_hello( + requested: Option<&str>, + protover: &mut u8, + is_authenticated: bool, + is_replica: bool, +) -> Vec { + if let Some(raw) = requested { + match raw.parse::() { + Ok(v @ (2 | 3)) => *protover = v, + _ => { + return Value::Error("NOPROTO unsupported protocol version".to_string()) + .serialize(); + } + } + } + + // Pre-auth HELLO reports the protocol but nothing about the server, so an + // unauthenticated client cannot use it to fingerprint the deployment. + if !is_authenticated { + return Value::Error("NOAUTH HELLO must be called with authentication".to_string()) + .serialize(); + } + + let fields = vec![ + ("server", Value::BulkString(Some(b"recached".to_vec()))), + ( + "version", + Value::BulkString(Some(env!("CARGO_PKG_VERSION").as_bytes().to_vec())), + ), + ("proto", Value::Integer(*protover as i64)), + ("mode", Value::BulkString(Some(b"standalone".to_vec()))), + ( + "role", + Value::BulkString(Some(if is_replica { + b"replica".to_vec() + } else { + b"master".to_vec() + })), + ), + ("modules", Value::Array(Some(vec![]))), + ]; + + if *protover >= 3 { + Value::Map( + fields + .into_iter() + .map(|(k, v)| (Value::BulkString(Some(k.as_bytes().to_vec())), v)) + .collect(), + ) + .serialize() + } else { + // RESP2 has no map type; Redis flattens to alternating key/value. + let mut flat = Vec::with_capacity(fields.len() * 2); + for (k, v) in fields { + flat.push(Value::BulkString(Some(k.as_bytes().to_vec()))); + flat.push(v); + } + Value::Array(Some(flat)).serialize() + } +} + +// ── INFO ───────────────────────────────────────────────────────────────────── + +/// Handles an AUTH attempt. Returns `(disconnect, resp_bytes)`. +/// +/// `disconnect` is true when the failure count hits MAX_AUTH_FAILURES. +pub(crate) fn process_auth( + provided: &str, + expected: &Arc>, + is_authenticated: &mut bool, + failures: &mut u32, +) -> (bool, Vec) { + match expected.as_ref() { + // Constant-time compare so a network attacker can't recover the password + // byte-by-byte from response-timing differences. + Some(pwd) if ct_eq_bytes(provided.as_bytes(), pwd.as_bytes()) => { + *is_authenticated = true; + *failures = 0; + (false, b"+OK\r\n".to_vec()) + } + Some(_) => { + *failures += 1; + if *failures >= MAX_AUTH_FAILURES { + (true, b"-ERR too many authentication failures\r\n".to_vec()) + } else { + (false, b"-ERR invalid password\r\n".to_vec()) + } + } + None => ( + false, + b"-ERR Client sent AUTH, but no password is set\r\n".to_vec(), + ), + } +} + +// ── main ───────────────────────────────────────────────────────────────────── + +#[allow(clippy::too_many_arguments)] +pub(crate) async fn handle_tcp( + socket: S, + store: Arc, + tx: broadcast::Sender, + password: Arc>, + pubsub: SharedPubSub, + watch_registry: WatchRegistry, + state: Arc, + peer: String, +) where + S: AsyncRead + AsyncWrite + Unpin + Send, +{ + let conn_id = next_conn_id(); + let mut client_meta = ClientMeta::new( + conn_id, + peer, + format!("127.0.0.1:{}", server_facts().tcp_port), + ); + let _guard = ConnectionGuard::new("tcp", client_meta.clone()); + let (mut reader, raw_writer) = tokio::io::split(socket); + let mut writer = tokio::io::BufWriter::with_capacity(32 * 1024, raw_writer); + let mut buf = Vec::::new(); + let mut read_pos: usize = 0; + // Bytes the in-flight frame needs before another parse attempt can + // possibly succeed; 0 means "try now". See the gate in the read arm. + let mut need: usize = 0; + let mut read_buf = [0u8; TCP_READ_BUFFER_BYTES]; + // Reused for every response on this connection — avoids a Vec allocation + // per command (significant under pipelining). + let mut resp_buf = Vec::::with_capacity(4 * 1024); + let mut is_authenticated = password.is_none(); + let mut auth_failures: u32 = 0; + // RESP2 until the client negotiates otherwise. Defaulting to 2 keeps every + // existing client working: they never send HELLO and must not start + // receiving RESP3-only types. + let mut protover: u8 = 2; + let mut multi_queue: Option> = None; + // Set when a command inside the open MULTI could not be queued. `EXEC` then + // refuses the whole transaction — see `EXECABORT`. + let mut multi_dirty = false; + let mut subscribed_channels: HashSet = HashSet::new(); + let mut subscribed_patterns: HashSet = HashSet::new(); + let (ps_tx, mut ps_rx) = mpsc::unbounded_channel::(); + // WATCH state for optimistic-lock transactions over TCP. Unlike the WS + // handler, TCP clients are not sent keychange pushes — WATCH is pure CAS. + let mut watched_keys: HashSet = HashSet::new(); + let mut watch_dirty = false; + let (watch_tx, mut watch_rx) = mpsc::unbounded_channel::(); + + 'outer: loop { + let is_subscribed = !subscribed_channels.is_empty() || !subscribed_patterns.is_empty(); + // Republish only when the counts moved: CLIENT LIST has to see live + // subscription state, but taking the registry's write lock once per + // command would put it on the hot path. + if client_meta.sub != subscribed_channels.len() + || client_meta.psub != subscribed_patterns.len() + { + client_meta.sub = subscribed_channels.len(); + client_meta.psub = subscribed_patterns.len(); + publish_client(client_meta.clone()); + } + + tokio::select! { + result = reader.read(&mut read_buf) => { + match result { + Ok(0) => break, + Ok(n) => { + if (buf.len() - read_pos) + n > MAX_TCP_READ_BUFFER_BYTES { + warn!("TCP connection exceeded max buffer size, closing"); + break 'outer; + } + buf.extend_from_slice(&read_buf[..n]); + // A frame that cannot possibly be complete is not worth + // re-parsing. `Value::parse` starts from the beginning + // every time, rebuilding — and reallocating — every + // bulk string it has already seen, so a large multi-bulk + // arriving over hundreds of segments used to re-copy + // everything received so far on each one. `need` is the + // parser's lower bound on the finished frame; until the + // buffer holds that much, skip the work entirely. + if buf.len() - read_pos < need { + continue 'outer; + } + 'parse: loop { + // Completeness is decided by the non-allocating + // measure. `Value::parse` restarts from the first + // byte every call, so asking *it* whether a frame + // had arrived meant rebuilding — and reallocating — + // every element received so far, once per segment, + // and discarding all of it. `frame_len` walks the + // headers and steps over payloads; `parse` below + // then runs once, on a frame known to be whole. + match Value::frame_len(&buf[read_pos..]) { + Ok(_) => {} + Err(e) if e.is_incomplete() => { + // Measured from `read_pos`, and compaction + // moves that to 0, so the bound stays valid. + need = e.needed(); + // Compact: drop already-parsed bytes. + buf.drain(..read_pos); + read_pos = 0; + break 'parse; + } + Err(e) => { + warn!("TCP protocol error: {}", e); + let _ = writer.write_all(b"-ERR Protocol error\r\n").await; + buf.clear(); + read_pos = 0; + need = 0; + break 'parse; + } + } + match Value::parse(&buf[read_pos..]) { + Ok((value, consumed)) => { + read_pos += consumed; + let cmd = match Command::from_value(value) { + Ok(c) => c, + Err(e) => { + // A frame that will not parse (bad arity, + // malformed argument) inside an open MULTI + // poisons the transaction, as in Redis. + if multi_queue.is_some() { multi_dirty = true; } + let r = Value::Error(e).serialize(); + if writer.write_all(&r).await.is_err() { break 'outer; } + continue 'parse; + } + }; + + // AUTH is always processed immediately + if let Command::Auth(ref pwd) = cmd { + let (disconnect, resp) = process_auth( + pwd, &password, &mut is_authenticated, &mut auth_failures, + ); + if writer.write_all(&resp).await.is_err() { break 'outer; } + if disconnect { + let _ = writer.flush().await; + break 'outer; + } + continue 'parse; + } + + if let Command::Hello(ref requested) = cmd { + let resp = process_hello( + requested.as_deref(), + &mut protover, + is_authenticated, + state.is_replica(), + ); + if writer.write_all(&resp).await.is_err() { break 'outer; } + // CLIENT LIST reports resp= per connection, + // so a renegotiation has to reach the registry. + if client_meta.resp != protover { + client_meta.resp = protover; + publish_client(client_meta.clone()); + } + continue 'parse; + } + + // QUIT is answered before the auth gate and + // before the subscribe-mode gate, as in Redis: + // a client that cannot authenticate, or is + // parked in subscribe mode, still deserves a + // clean close rather than a dropped socket. + if matches!(cmd, Command::Quit) { + let _ = writer.write_all(b"+OK\r\n").await; + let _ = writer.flush().await; + break 'outer; + } + + if !is_authenticated { + if writer.write_all(b"-NOAUTH Authentication required.\r\n").await.is_err() { + break 'outer; + } + continue 'parse; + } + + // ── Transactions ────────────────────────────── + match &cmd { + Command::Multi => { + let resp = if multi_queue.is_some() { + b"-ERR MULTI calls can not be nested\r\n".to_vec() + } else { + multi_queue = Some(Vec::new()); + multi_dirty = false; + b"+OK\r\n".to_vec() + }; + if writer.write_all(&resp).await.is_err() { break 'outer; } + continue 'parse; + } + Command::Discard => { + let resp = if multi_queue.take().is_some() { + // DISCARD also flushes WATCH state. + unregister_all_watches(&watch_registry, conn_id, &mut watched_keys).await; + while watch_rx.try_recv().is_ok() {} + watch_dirty = false; + multi_dirty = false; + b"+OK\r\n".to_vec() + } else { + b"-ERR DISCARD without MULTI\r\n".to_vec() + }; + if writer.write_all(&resp).await.is_err() { break 'outer; } + continue 'parse; + } + Command::Exec => { + match multi_queue.take() { + None => { + if writer.write_all(b"-ERR EXEC without MULTI\r\n").await.is_err() { break 'outer; } + } + // A command failed to queue: run nothing and + // say so, rather than applying the rest. + Some(_) if multi_dirty => { + multi_dirty = false; + unregister_all_watches(&watch_registry, conn_id, &mut watched_keys).await; + while watch_rx.try_recv().is_ok() {} + watch_dirty = false; + if writer.write_all(EXECABORT).await.is_err() { break 'outer; } + } + Some(queue) => { + // Drain pending notifications so the CAS check isn't racy. + while watch_rx.try_recv().is_ok() { + watch_dirty = true; + } + if watch_dirty { + // A watched key changed since WATCH — abort with nil array. + unregister_all_watches(&watch_registry, conn_id, &mut watched_keys).await; + while watch_rx.try_recv().is_ok() {} + watch_dirty = false; + if writer.write_all(&Value::Array(None).serialize()).await.is_err() { break 'outer; } + } else { + let mut results = Vec::with_capacity(queue.len()); + let armed = write_effects_armed(&tx, &state, &watch_registry); + for qcmd in queue { + let resp = match qcmd { + // Delivery lives in the connection loop, not the + // store — `store.execute(Publish)` is a stub that + // answers 0 and sends nothing. Queuing PUBLISH + // without this arm would silently swallow the + // message, which is worse than refusing it. + Command::Publish(ref channel, ref message) => { + let count = pubsub.lock().await.publish(channel, message); + Value::Integer(count) + } + _ if armed && is_write_command(&qcmd) => { + let resp = execute_and_record(&store, qcmd.clone()); + apply_write_effects(&qcmd, &resp, &tx, 0, &state, &watch_registry, &store).await; + resp + } + _ => execute_and_record(&store, qcmd), + }; + results.push(resp); + } + unregister_all_watches(&watch_registry, conn_id, &mut watched_keys).await; + while watch_rx.try_recv().is_ok() {} + watch_dirty = false; + let out = Value::Array(Some(results)).serialize(); + if writer.write_all(&out).await.is_err() { break 'outer; } + } + } + } + continue 'parse; + } + _ => {} + } + + // If inside MULTI, queue the command + if let Some(ref mut queue) = multi_queue { + // Every branch that does not reach `+QUEUED` also + // sets `multi_dirty`: the client asked for a + // transaction containing this command, and it will + // not be there, so running the remainder would + // apply something the caller never asked for. + // + // Refused here rather than at EXEC — an unknown + // verb must not sit in the queue looking accepted. + let refusal = queue_time_rejection(&cmd).or_else(|| match &cmd { + Command::Subscribe(_) | Command::Unsubscribe(_) + | Command::PSubscribe(_) | Command::PUnsubscribe(_) + | Command::Watch(_) | Command::Unwatch(_) + | Command::QSub(_) | Command::QUnsub(_) => Some( + b"-ERR Command not allowed inside a transaction\r\n".to_vec(), + ), + _ if queue.len() >= max_multi_queue_len() => Some( + b"-ERR transaction queue limit reached\r\n".to_vec(), + ), + _ => None, + }); + match refusal { + Some(err) => { + multi_dirty = true; + if writer.write_all(&err).await.is_err() { break 'outer; } + } + None => { + queue.push(cmd); + if writer.write_all(b"+QUEUED\r\n").await.is_err() { break 'outer; } + } + } + continue 'parse; + } + + // ── Pub/Sub commands ────────────────────────── + match cmd { + Command::Subscribe(channels) => { + for ch in channels { + subscribed_channels.insert(ch.clone()); + pubsub.lock().await.subscribe(conn_id, &ch, ps_tx.clone()); + let count = subscribed_channels.len() + subscribed_patterns.len(); + let ack = resp_subscribe_ack("subscribe", &ch, count); + if writer.write_all(&ack).await.is_err() { break 'outer; } + } + } + Command::Unsubscribe(channels) => { + let targets: Vec = if channels.is_empty() { + subscribed_channels.drain().collect() + } else { + channels.into_iter().filter(|c| subscribed_channels.remove(c)).collect() + }; + for ch in &targets { + pubsub.lock().await.unsubscribe(conn_id, ch); + let count = subscribed_channels.len() + subscribed_patterns.len(); + let ack = resp_subscribe_ack("unsubscribe", ch, count); + if writer.write_all(&ack).await.is_err() { break 'outer; } + } + if targets.is_empty() { + let ack = resp_subscribe_ack("unsubscribe", "", 0); + if writer.write_all(&ack).await.is_err() { break 'outer; } + } + } + Command::PSubscribe(patterns) => { + for pat in patterns { + subscribed_patterns.insert(pat.clone()); + pubsub.lock().await.psubscribe(conn_id, &pat, ps_tx.clone()); + let count = subscribed_channels.len() + subscribed_patterns.len(); + let ack = resp_subscribe_ack("psubscribe", &pat, count); + if writer.write_all(&ack).await.is_err() { break 'outer; } + } + } + Command::PUnsubscribe(patterns) => { + let targets: Vec = if patterns.is_empty() { + subscribed_patterns.drain().collect() + } else { + patterns.into_iter().filter(|p| subscribed_patterns.remove(p)).collect() + }; + for pat in &targets { + pubsub.lock().await.punsubscribe(conn_id, pat); + let count = subscribed_channels.len() + subscribed_patterns.len(); + let ack = resp_subscribe_ack("punsubscribe", pat, count); + if writer.write_all(&ack).await.is_err() { break 'outer; } + } + if targets.is_empty() { + let ack = resp_subscribe_ack("punsubscribe", "", 0); + if writer.write_all(&ack).await.is_err() { break 'outer; } + } + } + Command::Publish(channel, message) => { + let count = pubsub.lock().await.publish(&channel, &message); + let resp = Value::Integer(count).serialize(); + if writer.write_all(&resp).await.is_err() { break 'outer; } + } + + Command::Watch(keys) => { + let new_count = keys.iter().filter(|k| !watched_keys.contains(*k)).count(); + if watched_keys.len() + new_count > max_watches_per_conn() { + if writer.write_all(b"-ERR watch limit per connection reached\r\n").await.is_err() { break 'outer; } + } else { + { + let mut reg = watch_registry.map.lock().await; + for key in &keys { + if watched_keys.insert(key.clone()) { + reg.entry(key.clone()).or_default().push((conn_id, watch_tx.clone())); + } + } + watch_registry.sync_len(®); + } + if writer.write_all(b"+OK\r\n").await.is_err() { break 'outer; } + } + } + Command::Unwatch(keys) => { + let targets: Vec = if keys.is_empty() { + watched_keys.drain().collect() + } else { + keys.into_iter().filter(|k| watched_keys.remove(k)).collect() + }; + { + let mut reg = watch_registry.map.lock().await; + for key in &targets { + if let Some(subs) = reg.get_mut(key) { + subs.retain(|(id, _)| *id != conn_id); + if subs.is_empty() { reg.remove(key); } + } + } + watch_registry.sync_len(®); + } + if watched_keys.is_empty() { + while watch_rx.try_recv().is_ok() {} + watch_dirty = false; + } + if writer.write_all(b"+OK\r\n").await.is_err() { break 'outer; } + } + + cmd => { + // In subscribe mode only ping is allowed + if is_subscribed && !matches!(cmd, Command::Ping(_)) { + let err = b"-ERR only (P)SUBSCRIBE / (P)UNSUBSCRIBE / PING / QUIT allowed in subscribe mode\r\n"; + if writer.write_all(err).await.is_err() { break 'outer; } + continue 'parse; + } + // Replica: reject writes + if state.is_replica() && is_write_command(&cmd) { + let err = b"-READONLY You can't write against a read only replica.\r\n"; + if writer.write_all(err).await.is_err() { break 'outer; } + continue 'parse; + } + // Snapshot commands — handled here (async I/O, not in execute()) + match &cmd { + Command::Save => { + state.save(&store).await; + if writer.write_all(b"+OK\r\n").await.is_err() { break 'outer; } + continue 'parse; + } + Command::BgSave => { + let s = Arc::clone(&store); + let st = Arc::clone(&state); + tokio::spawn(async move { st.save(&s).await; }); + if writer.write_all(b"+Background saving started\r\n").await.is_err() { break 'outer; } + continue 'parse; + } + Command::LastSave => { + let ts = state.snap.last_save.load(Ordering::Relaxed); + if writer.write_all(&Value::Integer(ts).serialize()).await.is_err() { break 'outer; } + continue 'parse; + } + Command::Info(sections) => { + let repl = ReplInfo { + connected: state.replicas.count.load(Ordering::Relaxed), + queue_depth: state.replicas.max_queue_depth().await, + lag_frames: state.replicas.max_lag_frames().await, + }; + let body = render_info( + sections, + server_facts(), + &store, + sampled_keyspace(&store), + state.is_replica(), + repl, + state.snap.last_save.load(Ordering::Relaxed), + watch_registry.watched_patterns.load(Ordering::Relaxed) as u64, + watch_registry.watched_keys.load(Ordering::Relaxed) as u64, + ); + let resp = Value::BulkString(Some(body.into_bytes())).serialize(); + if writer.write_all(&resp).await.is_err() { break 'outer; } + continue 'parse; + } + Command::Client(args) => { + let resp = handle_client_command(args, &mut client_meta).serialize(); + if writer.write_all(&resp).await.is_err() { break 'outer; } + continue 'parse; + } + Command::Config(args) => { + let resp = handle_config_command(args, server_facts(), &store).serialize(); + if writer.write_all(&resp).await.is_err() { break 'outer; } + continue 'parse; + } + Command::CommandQuery(args) => { + let resp = handle_command_query(args, protover).serialize(); + if writer.write_all(&resp).await.is_err() { break 'outer; } + continue 'parse; + } + Command::Cluster(args) => { + let resp = handle_cluster_command(args).serialize(); + if writer.write_all(&resp).await.is_err() { break 'outer; } + continue 'parse; + } + Command::Module(args) => { + let resp = handle_module_command(args).serialize(); + if writer.write_all(&resp).await.is_err() { break 'outer; } + continue 'parse; + } + Command::Memory(args) => { + let resp = handle_memory_command(args).serialize(); + if writer.write_all(&resp).await.is_err() { break 'outer; } + continue 'parse; + } + Command::PubSub(args) => { + let resp = handle_pubsub_command(args, &*pubsub.lock().await).serialize(); + if writer.write_all(&resp).await.is_err() { break 'outer; } + continue 'parse; + } + Command::ReplicaOfNoOne => { + state.promote_to_primary(); + if writer.write_all(b"+OK\r\n").await.is_err() { break 'outer; } + continue 'parse; + } + _ => {} + } + let response = if is_write_command(&cmd) + && write_effects_armed(&tx, &state, &watch_registry) + { + let response = execute_and_record(&store, cmd.clone()); + apply_write_effects(&cmd, &response, &tx, 0, &state, &watch_registry, &store).await; + response + } else { + execute_and_record(&store, cmd) + }; + resp_buf.clear(); + response.serialize_into(&mut resp_buf); + if writer.write_all(&resp_buf).await.is_err() { + break 'outer; + } + } + } + } + Err(e) if e.is_incomplete() => { + // Measured from `read_pos`, and compaction + // moves that to 0, so the bound stays valid. + need = e.needed(); + // Compact: drop already-parsed bytes, reset cursor. + buf.drain(..read_pos); + read_pos = 0; + break 'parse; + } + Err(e) => { + warn!("TCP protocol error: {}", e); + let _ = writer.write_all(b"-ERR Protocol error\r\n").await; + buf.clear(); + read_pos = 0; + need = 0; + break 'parse; + } + } + } + // Flush all responses for this read batch in one syscall. + if writer.flush().await.is_err() { + break 'outer; + } + } + Err(e) => { + warn!("TCP read error: {}", e); + break; + } + } + } + + msg = ps_rx.recv(), if is_subscribed => { + match msg { + Some(m) => { + if writer.write_all(&encode_pubsub_msg(m, protover)).await.is_err() { + break; + } + // `writer` is a BufWriter, and a delivery is not a + // response to anything this connection sent — nothing + // else is going to flush it. Without this a subscriber + // that only listens receives nothing until it happens + // to send a command or 32 KB of pushes accumulate. + if writer.flush().await.is_err() { + break; + } + } + None => break, + } + } + + // A watched key changed: mark the transaction dirty so a following + // EXEC aborts. TCP clients get no keychange push (WATCH is pure CAS). + notif = watch_rx.recv(), if !watched_keys.is_empty() => { + if notif.is_some() { + watch_dirty = true; + } + } + } + } + + if !subscribed_channels.is_empty() || !subscribed_patterns.is_empty() { + pubsub.lock().await.unsubscribe_all(conn_id); + } + unregister_all_watches(&watch_registry, conn_id, &mut watched_keys).await; +} + +// ── WebSocket handler ───────────────────────────────────────────────────────── + +/// Complete the WebSocket handshake, enforcing the origin allowlist and a +/// deadline. Returns `None` when the connection was refused, failed, or stalled +/// — in every case the caller simply drops the socket and its permit. +/// +/// Split out from `handle_ws` so both the origin decision and the timeout are +/// reachable from a test without standing up a listener. +/// +/// `result_large_err`: the error type is tungstenite's `ErrorResponse`, which is +/// an `http::Response` — its size is the handshake callback's signature, not +/// ours, and boxing it would not satisfy the trait. +#[allow(clippy::result_large_err)] +pub(crate) async fn ws_handshake( + socket: S, + allowed_origins: Option<&[String]>, + timeout: Duration, + conn_id: u64, +) -> Option> +where + S: AsyncRead + AsyncWrite + Unpin, +{ + let check_origin = |req: &HandshakeRequest, + resp: HandshakeResponse| + -> Result { + let origin = req + .headers() + .get("origin") + .and_then(|v| v.to_str().ok()) + .map(str::to_owned); + if origin_allowed(allowed_origins, origin.as_deref()) { + return Ok(resp); + } + warn!( + "WS conn {}: refused Origin {:?} — not in RECACHED_ALLOWED_ORIGINS", + conn_id, origin + ); + let mut err = ErrorResponse::new(Some( + "Origin not allowed. Add it to RECACHED_ALLOWED_ORIGINS to permit this page." + .to_string(), + )); + *err.status_mut() = StatusCode::FORBIDDEN; + Err(err) + }; + + match tokio::time::timeout(timeout, accept_hdr_async(socket, check_origin)).await { + Ok(Ok(ws)) => Some(ws), + Ok(Err(e)) => { + warn!("WS handshake failed on conn {}: {}", conn_id, e); + None + } + Err(_) => { + debug!("WS handshake on conn {} timed out", conn_id); + None + } + } +} + +#[allow(clippy::too_many_arguments)] +pub(crate) async fn handle_ws( + socket: S, + store: Arc, + tx: broadcast::Sender, + password: Arc>, + conn_id: u64, + pubsub: SharedPubSub, + watch_registry: WatchRegistry, + state: Arc, + sync_secret: Arc>, + allowed_origins: Arc>>, + peer: String, +) where + S: AsyncRead + AsyncWrite + Unpin + Send, +{ + let mut client_meta = ClientMeta::new( + conn_id, + peer, + format!("127.0.0.1:{}", server_facts().ws_port), + ); + // The WebSocket transport speaks RESP3 from the first frame. + client_meta.resp = 3; + let _guard = ConnectionGuard::new("ws", client_meta.clone()); + let Some(ws_stream) = ws_handshake( + socket, + allowed_origins.as_deref(), + handshake_timeout(), + conn_id, + ) + .await + else { + return; + }; + + let (mut ws_sender, mut ws_receiver) = ws_stream.split(); + let mut rx = tx.subscribe(); + let mut is_authenticated = password.is_none(); + let mut auth_failures: u32 = 0; + let mut multi_queue: Option> = None; + // Set when a command inside the open MULTI could not be queued. `EXEC` then + // refuses the whole transaction — see `EXECABORT`. + let mut multi_dirty = false; + let mut subscribed_channels: HashSet = HashSet::new(); + let mut subscribed_patterns: HashSet = HashSet::new(); + let (ps_tx, mut ps_rx) = mpsc::unbounded_channel::(); + let mut watched_keys: HashSet = HashSet::new(); + // Set when any watched key changes; EXEC aborts (returns nil) if true. + let mut watch_dirty = false; + let (watch_tx, mut watch_rx) = mpsc::unbounded_channel::(); + // Sync scopes for this connection. `strict` (RECACHED_SYNC_SECRET set) + // means: no pushes and no key commands until a signed token is presented. + // Without a secret, scopes are an opt-in bandwidth filter (legacy fan-out + // of everything when None). + let strict = sync_secret.is_some(); + let mut sync_scopes: Option> = None; + // Live-query subscriptions (QSUB). Keychange notifications for matching + // keys arrive on their own channel so they never dirty WATCH transactions. + let mut qsub_patterns: HashSet = HashSet::new(); + let (q_tx, mut q_rx) = mpsc::unbounded_channel::(); + + // Replies go out as *text* frames whenever the RESP bytes are valid UTF-8, + // which is the overwhelming majority and is what every existing client + // expects. A reply carrying a value that is not valid UTF-8 goes out as a + // *binary* frame instead of being mangled by a lossy conversion, which is + // what made raw binary values round-trip only over the TCP port. + macro_rules! ws_send { + ($bytes:expr) => {{ + let bytes: &[u8] = $bytes; + let msg = match std::str::from_utf8(bytes) { + Ok(text) => Message::Text(text.into()), + Err(_) => Message::Binary(bytes.to_vec().into()), + }; + if ws_sender.send(msg).await.is_err() { + break; + } + }}; + } + + 'outer: loop { + let is_subscribed = !subscribed_channels.is_empty() || !subscribed_patterns.is_empty(); + + tokio::select! { + msg = ws_receiver.next() => { + match msg { + // Binary frames carry the same RESP bytes as text frames. + // They exist so a client can write a value that is not + // valid UTF-8 — impossible over a text frame, which the + // WebSocket spec requires to be well-formed UTF-8. + Some(Ok(frame @ (Message::Text(_) | Message::Binary(_)))) => { + let raw: Vec = match &frame { + Message::Text(t) => t.as_bytes().to_vec(), + Message::Binary(b) => b.to_vec(), + _ => unreachable!("pattern restricts to text and binary"), + }; + let (value, _) = match Value::parse(&raw) { + Ok(v) => v, + Err(e) => { + let err = Value::Error(format!("ERR Protocol error: {}", e)).serialize(); + ws_send!(&err); + continue; + } + }; + + let cmd = match Command::from_value(value) { + Ok(c) => c, + Err(e) => { + // A frame that will not parse inside an open MULTI + // poisons the transaction, as in Redis. + if multi_queue.is_some() { multi_dirty = true; } + let err = Value::Error(e).serialize(); + ws_send!(&err); + continue; + } + }; + + // AUTH + if let Command::Auth(ref pwd) = cmd { + let (disconnect, resp) = process_auth( + pwd, &password, &mut is_authenticated, &mut auth_failures, + ); + ws_send!(&resp); + if disconnect { break; } + continue; + } + + // The WebSocket sync protocol is specified in terms of + // RESP3 push frames, so this transport is always RESP3 + // and HELLO cannot downgrade it — a client asking for 2 + // is refused rather than silently left on 3. + if let Command::Hello(ref requested) = cmd { + let mut ws_protover: u8 = 3; + let resp = match requested.as_deref() { + Some("2") => Value::Error( + "NOPROTO the WebSocket transport requires RESP3".to_string(), + ) + .serialize(), + other => process_hello( + other, + &mut ws_protover, + is_authenticated, + state.is_replica(), + ), + }; + ws_send!(&resp); + continue; + } + + if matches!(cmd, Command::Quit) { + ws_send!(b"+OK\r\n"); + break 'outer; + } + + if !is_authenticated { + let resp = Value::Error("NOAUTH Authentication required.".to_string()).serialize(); + ws_send!(&resp); + continue; + } + + // ── Sync scoping ────────────────────────────────────── + if let Command::Sync(ref args) = cmd { + let resp = handle_sync_command(args, (*sync_secret).as_deref(), &mut sync_scopes, conn_id); + ws_send!(&resp); + continue; + } + // Token-scoped mode: check every command against this + // connection's granted scopes before it runs (including + // commands about to be queued inside MULTI). + if strict { + match command_scope(&cmd) { + CommandScope::KeyLess => {} + CommandScope::Admin => { + ws_send!(b"-NOSCOPE keyspace-wide and administrative commands are not available on scoped WebSocket connections\r\n"); + continue; + } + CommandScope::Keys(keys) => { + let Some(ref scopes) = sync_scopes else { + ws_send!(b"-NOSCOPE send SYNC TOKEN before issuing commands\r\n"); + continue; + }; + if let Some(denied) = keys + .iter() + .find(|k| !scopes_match(scopes, std::slice::from_ref(k))) + { + let err = Value::Error(format!( + "NOSCOPE key '{}' is outside this connection's sync scopes", + denied + )) + .serialize(); + ws_send!(&err); + continue; + } + } + } + } + + // ── Transactions ────────────────────────────────────── + match &cmd { + Command::Multi => { + let resp = if multi_queue.is_some() { + b"-ERR MULTI calls can not be nested\r\n".to_vec() + } else { + multi_queue = Some(Vec::new()); + multi_dirty = false; + b"+OK\r\n".to_vec() + }; + ws_send!(&resp); + continue; + } + Command::Discard => { + let resp = if multi_queue.take().is_some() { + // DISCARD also flushes WATCH state. + unregister_all_watches(&watch_registry, conn_id, &mut watched_keys).await; + while watch_rx.try_recv().is_ok() {} // drop stale notifications + watch_dirty = false; + multi_dirty = false; + b"+OK\r\n".to_vec() + } else { + b"-ERR DISCARD without MULTI\r\n".to_vec() + }; + ws_send!(&resp); + continue; + } + Command::Exec => { + match multi_queue.take() { + None => { + ws_send!(b"-ERR EXEC without MULTI\r\n"); + } + // A command failed to queue: run nothing and + // say so, rather than applying the rest. + Some(_) if multi_dirty => { + multi_dirty = false; + unregister_all_watches(&watch_registry, conn_id, &mut watched_keys).await; + while watch_rx.try_recv().is_ok() {} + watch_dirty = false; + ws_send!(EXECABORT); + } + Some(queue) => { + // Catch watched-key changes that arrived but the select + // loop hasn't drained yet, so the CAS check isn't racy. + while watch_rx.try_recv().is_ok() { + watch_dirty = true; + } + if watch_dirty { + // A watched key changed since WATCH — abort: return + // a nil array and run nothing (Redis CAS semantics). + unregister_all_watches(&watch_registry, conn_id, &mut watched_keys).await; + while watch_rx.try_recv().is_ok() {} // drop stale notifications + watch_dirty = false; + ws_send!(&Value::Array(None).serialize()); + } else { + let mut results = Vec::with_capacity(queue.len()); + let armed = write_effects_armed(&tx, &state, &watch_registry); + for qcmd in queue { + let resp = match qcmd { + // See the TCP path: delivery lives here, + // not in the store. + Command::Publish(ref channel, ref message) => { + let count = pubsub.lock().await.publish(channel, message); + Value::Integer(count) + } + _ if armed && is_write_command(&qcmd) => { + let resp = execute_and_record(&store, qcmd.clone()); + apply_write_effects(&qcmd, &resp, &tx, conn_id, &state, &watch_registry, &store).await; + resp + } + _ => execute_and_record(&store, qcmd), + }; + results.push(resp); + } + // EXEC always flushes WATCH state. Drain any + // self-notifications the queued writes produced so + // they can't dirty a later transaction. + unregister_all_watches(&watch_registry, conn_id, &mut watched_keys).await; + while watch_rx.try_recv().is_ok() {} + watch_dirty = false; + let out = Value::Array(Some(results)).serialize(); + ws_send!(&out); + } + } + } + continue; + } + _ => {} + } + + // Queue if inside MULTI + if let Some(ref mut queue) = multi_queue { + // Mirrors the TCP path: anything that does not reach + // `+QUEUED` poisons the transaction so EXEC runs nothing. + let refusal = queue_time_rejection(&cmd).or_else(|| match &cmd { + Command::Subscribe(_) | Command::Unsubscribe(_) + | Command::PSubscribe(_) | Command::PUnsubscribe(_) + | Command::Watch(_) | Command::Unwatch(_) + | Command::QSub(_) | Command::QUnsub(_) => Some( + b"-ERR Command not allowed inside a transaction\r\n".to_vec(), + ), + _ if queue.len() >= max_multi_queue_len() => Some( + b"-ERR transaction queue limit reached\r\n".to_vec(), + ), + _ => None, + }); + match refusal { + Some(err) => { + multi_dirty = true; + ws_send!(&err); + } + None => { + queue.push(cmd); + ws_send!(b"+QUEUED\r\n"); + } + } + continue; + } + + // ── Pub/Sub commands ────────────────────────────────── + match cmd { + Command::Subscribe(channels) => { + for ch in channels { + subscribed_channels.insert(ch.clone()); + pubsub.lock().await.subscribe(conn_id, &ch, ps_tx.clone()); + let count = subscribed_channels.len() + subscribed_patterns.len(); + ws_send!(&resp_subscribe_ack("subscribe", &ch, count)); + } + } + Command::Unsubscribe(channels) => { + let targets: Vec = if channels.is_empty() { + subscribed_channels.drain().collect() + } else { + channels.into_iter().filter(|c| subscribed_channels.remove(c)).collect() + }; + for ch in &targets { + pubsub.lock().await.unsubscribe(conn_id, ch); + let count = subscribed_channels.len() + subscribed_patterns.len(); + ws_send!(&resp_subscribe_ack("unsubscribe", ch, count)); + } + if targets.is_empty() { + ws_send!(&resp_subscribe_ack("unsubscribe", "", 0)); + } + } + Command::PSubscribe(patterns) => { + for pat in patterns { + subscribed_patterns.insert(pat.clone()); + pubsub.lock().await.psubscribe(conn_id, &pat, ps_tx.clone()); + let count = subscribed_channels.len() + subscribed_patterns.len(); + ws_send!(&resp_subscribe_ack("psubscribe", &pat, count)); + } + } + Command::PUnsubscribe(patterns) => { + let targets: Vec = if patterns.is_empty() { + subscribed_patterns.drain().collect() + } else { + patterns.into_iter().filter(|p| subscribed_patterns.remove(p)).collect() + }; + for pat in &targets { + pubsub.lock().await.punsubscribe(conn_id, pat); + let count = subscribed_channels.len() + subscribed_patterns.len(); + ws_send!(&resp_subscribe_ack("punsubscribe", pat, count)); + } + if targets.is_empty() { + ws_send!(&resp_subscribe_ack("punsubscribe", "", 0)); + } + } + Command::Publish(channel, message) => { + let count = pubsub.lock().await.publish(&channel, &message); + ws_send!(&Value::Integer(count).serialize()); + } + + Command::Watch(keys) => { + let new_count = keys + .iter() + .filter(|k| !watched_keys.contains(*k)) + .count(); + if watched_keys.len() + new_count > max_watches_per_conn() { + ws_send!(b"-ERR watch limit per connection reached\r\n"); + } else { + { + let mut reg = watch_registry.map.lock().await; + for key in &keys { + if watched_keys.insert(key.clone()) { + reg.entry(key.clone()) + .or_default() + .push((conn_id, watch_tx.clone())); + } + } + watch_registry.sync_len(®); + } // reg dropped before await + ws_send!(b"+OK\r\n"); + } + } + Command::Unwatch(keys) => { + let targets: Vec = if keys.is_empty() { + watched_keys.drain().collect() + } else { + keys.into_iter().filter(|k| watched_keys.remove(k)).collect() + }; + { + let mut reg = watch_registry.map.lock().await; + for key in &targets { + if let Some(subs) = reg.get_mut(key) { + subs.retain(|(id, _)| *id != conn_id); + if subs.is_empty() { + reg.remove(key); + } + } + } + watch_registry.sync_len(®); + } + // Once nothing is watched, clear the dirty flag and drop any + // queued notifications so a later WATCH/MULTI/EXEC starts clean. + if watched_keys.is_empty() { + while watch_rx.try_recv().is_ok() {} + watch_dirty = false; + } + ws_send!(b"+OK\r\n"); + } + + Command::QSub(pattern) => { + // Strict mode: the requested pattern must sit inside a + // granted scope. A grant covers the request when it is + // identical or glob-matches the request as literal text + // (prefix-style grants: `cart:*` covers `cart:42:*`). + if strict { + let allowed = sync_scopes.as_ref().is_some_and(|scopes| { + scopes.iter().any(|s| { + s == &pattern + || core_engine::store::glob_match(s, &pattern) + }) + }); + if !allowed { + ws_send!(b"-NOSCOPE pattern is outside this connection's sync scopes\r\n"); + continue 'outer; + } + } + if !qsub_patterns.contains(&pattern) + && qsub_patterns.len() >= max_qsubs_per_conn() + { + ws_send!(b"-ERR live query limit per connection reached\r\n"); + continue 'outer; + } + // Register *before* snapshotting: a write landing in + // between is delivered as a keychange after the initial + // state, which is idempotent — the reverse order would + // lose it. + if qsub_patterns.insert(pattern.clone()) { + let mut pats = watch_registry.patterns.lock().await; + pats.entry(pattern.clone()) + .or_default() + .push((conn_id, q_tx.clone())); + watch_registry.sync_patterns_len(&pats); + } + let kvs = store.matching_key_values(&pattern, max_qsub_initial_keys()); + // Tagged reply so clients can recognise it among + // interleaved frames: ["qstate", pattern, k, v, ...] + let mut items = Vec::with_capacity(kvs.len() * 2 + 2); + items.push(Value::BulkString(Some(b"qstate".to_vec()))); + items.push(Value::BulkString(Some(pattern.clone().into_bytes()))); + for (k, v) in kvs { + items.push(Value::BulkString(Some(k.into_bytes()))); + items.push(v); + } + ws_send!(&Value::Array(Some(items)).serialize()); + } + Command::QUnsub(pattern) => { + let targets: Vec = match pattern { + Some(p) => { + if qsub_patterns.remove(&p) { + vec![p] + } else { + vec![] + } + } + None => qsub_patterns.drain().collect(), + }; + if !targets.is_empty() { + let mut pats = watch_registry.patterns.lock().await; + for p in &targets { + if let Some(subs) = pats.get_mut(p) { + subs.retain(|(id, _)| *id != conn_id); + if subs.is_empty() { + pats.remove(p); + } + } + } + watch_registry.sync_patterns_len(&pats); + } + ws_send!(b"+OK\r\n"); + } + + cmd => { + // Exactly-once: unwrap the DEDUP envelope. An id at or + // below this client's high-water mark was already applied + // (its acknowledgment was lost) — skip it. +DUP still + // acknowledges the write so the client retires it. + let cmd = match cmd { + Command::Dedup(client, id, inner) => { + if state.dedup_seen(&client, id) { + ws_send!(b"+DUP\r\n"); + continue 'outer; + } + *inner + } + other => other, + }; + if is_subscribed && !matches!(cmd, Command::Ping(_)) { + ws_send!(b"-ERR only (P)SUBSCRIBE / (P)UNSUBSCRIBE / PING / QUIT allowed in subscribe mode\r\n"); + continue 'outer; + } + // Replica: reject writes + if state.is_replica() && is_write_command(&cmd) { + ws_send!(b"-READONLY You can't write against a read only replica.\r\n"); + continue 'outer; + } + // Snapshot commands + match &cmd { + Command::Save => { + state.save(&store).await; + ws_send!(b"+OK\r\n"); + continue 'outer; + } + Command::BgSave => { + let s = Arc::clone(&store); + let st = Arc::clone(&state); + tokio::spawn(async move { st.save(&s).await; }); + ws_send!(b"+Background saving started\r\n"); + continue 'outer; + } + Command::LastSave => { + let ts = state.snap.last_save.load(Ordering::Relaxed); + ws_send!(&Value::Integer(ts).serialize()); + continue 'outer; + } + Command::Info(sections) => { + let repl = ReplInfo { + connected: state.replicas.count.load(Ordering::Relaxed), + queue_depth: state.replicas.max_queue_depth().await, + lag_frames: state.replicas.max_lag_frames().await, + }; + let body = render_info( + sections, + server_facts(), + &store, + sampled_keyspace(&store), + state.is_replica(), + repl, + state.snap.last_save.load(Ordering::Relaxed), + watch_registry.watched_patterns.load(Ordering::Relaxed) as u64, + watch_registry.watched_keys.load(Ordering::Relaxed) as u64, + ); + ws_send!(&Value::BulkString(Some(body.into_bytes())).serialize()); + continue 'outer; + } + Command::Client(args) => { + ws_send!(&handle_client_command(args, &mut client_meta).serialize()); + continue 'outer; + } + Command::Config(args) => { + ws_send!(&handle_config_command(args, server_facts(), &store).serialize()); + continue 'outer; + } + Command::CommandQuery(args) => { + // The WebSocket transport is RESP3-only, + // so the catalog always replies as a map. + ws_send!(&handle_command_query(args, 3).serialize()); + continue 'outer; + } + Command::Cluster(args) => { + ws_send!(&handle_cluster_command(args).serialize()); + continue 'outer; + } + Command::Module(args) => { + ws_send!(&handle_module_command(args).serialize()); + continue 'outer; + } + Command::Memory(args) => { + ws_send!(&handle_memory_command(args).serialize()); + continue 'outer; + } + Command::PubSub(args) => { + ws_send!(&handle_pubsub_command(args, &*pubsub.lock().await).serialize()); + continue 'outer; + } + Command::ReplicaOfNoOne => { + state.promote_to_primary(); + ws_send!(b"+OK\r\n"); + continue 'outer; + } + _ => {} + } + // Ephemeral keys are owned by the connection that wrote + // them until another claims them; the close handler deletes + // whatever is still ours. Claimed outside the write-effects + // branch below, which only runs when a peer, replica, AOF or + // watcher is present — ownership must be recorded even on a + // standalone server with no listeners. + if let Command::ESet(ref k, _) = cmd { + state.claim_ephemeral(k, conn_id); + } + let response = if is_write_command(&cmd) + && write_effects_armed(&tx, &state, &watch_registry) + { + let response = execute_and_record(&store, cmd.clone()); + apply_write_effects(&cmd, &response, &tx, conn_id, &state, &watch_registry, &store).await; + response + } else { + execute_and_record(&store, cmd) + }; + ws_send!(&response.serialize()); + } + } + } + Some(Ok(_)) => {} + Some(Err(e)) => { + warn!("WS error on conn {}: {}", conn_id, e); + break; + } + None => break, + } + } + + result = rx.recv() => { + match result { + Ok(push) if push.origin != conn_id => { + // Scope filter: with scopes set, only matching keys are + // forwarded. Without scopes, legacy mode forwards + // everything; strict mode forwards nothing until a + // token has been presented. + let visible = match &sync_scopes { + Some(scopes) => scopes_match(scopes, &push.keys), + None => !strict, + }; + if visible { + ws_send!(&push.resp); + } + } + Ok(_) => {} + Err(broadcast::error::RecvError::Lagged(n)) => { + warn!("WS conn {} lagged, missed {} messages, resubscribing", conn_id, n); + rx = tx.subscribe(); + } + Err(broadcast::error::RecvError::Closed) => break, + } + } + + msg = ps_rx.recv(), if is_subscribed => { + match msg { + Some(m) => { + let bytes = encode_pubsub_msg(m, 3); + ws_send!(&bytes); + } + None => break, + } + } + + notif = watch_rx.recv(), if !watched_keys.is_empty() => { + if let Some((key, value)) = notif { + // A watched key changed: mark the transaction dirty (so a + // following EXEC aborts) and still push the keychange to the + // client for the observable-keys feature. + watch_dirty = true; + let bytes = encode_keychange(&key, &value); + ws_send!(&bytes); + } + } + + // Live-query keychange: same frame as WATCH pushes, but never + // dirties transactions. + notif = q_rx.recv(), if !qsub_patterns.is_empty() => { + if let Some((key, value)) = notif { + let bytes = encode_keychange(&key, &value); + ws_send!(&bytes); + } + } + } + } + + if !subscribed_channels.is_empty() || !subscribed_patterns.is_empty() { + pubsub.lock().await.unsubscribe_all(conn_id); + } + if !watched_keys.is_empty() { + let mut reg = watch_registry.map.lock().await; + for key in &watched_keys { + if let Some(subs) = reg.get_mut(key) { + subs.retain(|(id, _)| *id != conn_id); + if subs.is_empty() { + reg.remove(key); + } + } + } + watch_registry.sync_len(®); + } + unregister_all_qsubs(&watch_registry, conn_id, &mut qsub_patterns).await; + + // Delete ephemeral keys this connection still owns and fan the deletions + // out, so every subscriber sees the peer go away immediately rather than + // waiting for a heartbeat TTL to lapse. + let expired = state.take_ephemeral_for(conn_id); + if !expired.is_empty() { + let del = Command::Del(expired); + let response = store.execute(del.clone()); + apply_write_effects( + &del, + &response, + &tx, + conn_id, + &state, + &watch_registry, + &store, + ) + .await; + } +} diff --git a/server-native/src/info.rs b/server-native/src/info.rs new file mode 100644 index 0000000..a5acbd2 --- /dev/null +++ b/server-native/src/info.rs @@ -0,0 +1,749 @@ +//! Server introspection: INFO, CLIENT, CONFIG, COMMAND, CLUSTER, MODULE, +//! MEMORY and PUBSUB — everything reporting on the server rather than the data. + +use crate::*; + +/// Redis compatibility level advertised as `redis_version`. +/// +/// Clients feature-gate on this field, so it cannot be Recached's own version: +/// a library seeing `redis_version:0.2.3` concludes the server predates +/// everything and disables features it could safely use. 6.2 is the honest +/// floor — RESP3 and `HELLO` exist there, which Recached implements, while +/// nothing in 7.x that Recached lacks (functions, `OBJECT FREQ`, sharded +/// pub/sub) gets advertised. The real version ships alongside it as +/// `recached_version`, the same split KeyDB and Dragonfly use. +pub(crate) const REDIS_COMPAT_VERSION: &str = "6.2.0"; + +/// Sections `INFO` reports when called with no arguments. +pub(crate) const DEFAULT_INFO_SECTIONS: &[&str] = &[ + "server", + "clients", + "memory", + "persistence", + "stats", + "replication", + "cluster", + "keyspace", + "recached", +]; + +/// Process-wide startup facts, set once by `main`. +/// +/// Threaded through a static rather than the connection-handler signatures: +/// these values are immutable for the life of the process and needed only by +/// `INFO`, and the handlers already carry a long parameter list. Tests that +/// exercise `render_info` build their own `ServerFacts` and never touch this. +pub(crate) static SERVER_FACTS: std::sync::OnceLock = std::sync::OnceLock::new(); + +pub(crate) fn server_facts() -> &'static ServerFacts { + SERVER_FACTS.get_or_init(ServerFacts::default) +} + +/// Startup facts `INFO` reports that are fixed for the life of the process. +/// Captured in `main` once rather than re-read from the environment per call. +#[derive(Clone, Debug)] +pub(crate) struct ServerFacts { + pub(crate) start: SystemTime, + /// Random per-process identifier, as Redis reports it: 40 hex chars. + pub(crate) run_id: String, + pub(crate) tcp_port: u16, + pub(crate) ws_port: u16, + pub(crate) max_connections: usize, + pub(crate) tls_enabled: bool, + pub(crate) auth_enabled: bool, + pub(crate) aof_enabled: bool, +} + +impl Default for ServerFacts { + fn default() -> Self { + Self { + start: SystemTime::now(), + run_id: String::new(), + tcp_port: 6379, + ws_port: 6380, + max_connections: DEFAULT_MAX_CONNECTIONS, + tls_enabled: false, + auth_enabled: false, + aof_enabled: false, + } + } +} + +pub(crate) fn generate_run_id() -> String { + use rand::Rng; + let mut rng = rand::rng(); + (0..40) + .map(|_| std::char::from_digit(rng.random_range(0..16), 16).unwrap_or('0')) + .collect() +} + +/// Replication numbers, resolved by the caller. +/// +/// The registry's depth and lag accessors are async (they lock per-replica +/// queues), and `render_info` is a pure synchronous formatter so it stays +/// trivially testable — so the caller awaits them and passes the results in. +#[derive(Clone, Copy, Debug, Default)] +pub(crate) struct ReplInfo { + pub(crate) connected: usize, + pub(crate) queue_depth: usize, + pub(crate) lag_frames: u64, +} + +/// Last keyspace walk, refreshed every 5s by the metrics sampler. +/// +/// `keyspace_sample()` is O(keyspace). A monitoring agent polling `INFO` once a +/// second must not walk every key each time, so `INFO` reports the sample +/// instead. `u64::MAX` means "not sampled yet" — `INFO` then walks once itself, +/// which only happens in the first few seconds of uptime. +pub(crate) static SAMPLED_KEYS: AtomicU64 = AtomicU64::new(u64::MAX); + +pub(crate) static SAMPLED_VOLATILE_KEYS: AtomicU64 = AtomicU64::new(u64::MAX); + +pub(crate) static SAMPLED_MEMORY_BYTES: AtomicU64 = AtomicU64::new(u64::MAX); + +pub(crate) fn store_sampled_keyspace(sample: KeyspaceSample) { + SAMPLED_KEYS.store(sample.keys as u64, Ordering::Relaxed); + SAMPLED_VOLATILE_KEYS.store(sample.volatile_keys as u64, Ordering::Relaxed); + SAMPLED_MEMORY_BYTES.store(sample.memory_bytes as u64, Ordering::Relaxed); +} + +pub(crate) fn sampled_keyspace(store: &KeyValueStore) -> KeyspaceSample { + let keys = SAMPLED_KEYS.load(Ordering::Relaxed); + if keys == u64::MAX { + // Sampler has not run yet — walk once so the first INFO is not blank. + let sample = store.keyspace_sample(); + store_sampled_keyspace(sample); + return sample; + } + KeyspaceSample { + keys: keys as usize, + volatile_keys: SAMPLED_VOLATILE_KEYS.load(Ordering::Relaxed) as usize, + memory_bytes: SAMPLED_MEMORY_BYTES.load(Ordering::Relaxed) as usize, + } +} + +/// Render bytes the way Redis does for `*_human` fields. +pub(crate) fn human_bytes(bytes: u64) -> String { + const UNITS: [(&str, f64); 4] = [ + ("G", 1024.0 * 1024.0 * 1024.0), + ("M", 1024.0 * 1024.0), + ("K", 1024.0), + ("B", 1.0), + ]; + for (suffix, size) in UNITS { + if bytes as f64 >= size { + return if suffix == "B" { + format!("{}B", bytes) + } else { + format!("{:.2}{}", bytes as f64 / size, suffix) + }; + } + } + "0B".to_string() +} + +pub(crate) fn eviction_policy_name(policy: EvictionPolicy) -> &'static str { + match policy { + EvictionPolicy::NoEviction => "noeviction", + EvictionPolicy::AllKeysLru => "allkeys-lru", + EvictionPolicy::AllKeysRandom => "allkeys-random", + EvictionPolicy::VolatileLru => "volatile-lru", + EvictionPolicy::VolatileTtl => "volatile-ttl", + } +} + +// ── CLIENT / CONFIG / COMMAND ───────────────────────────────────────────────── + +/// Wrong-subcommand error in Redis's wording, which some clients match on. +pub(crate) fn unknown_subcommand(container: &str, sub: &str) -> Value { + Value::Error(format!( + "ERR Unknown subcommand or wrong number of arguments for '{sub}'. \ + Try {container} HELP." + )) +} + +/// Handle `CLIENT `, mutating this connection's `meta` in place. +/// +/// Returns `None` for subcommands Recached does not implement, so the caller +/// can answer with the standard error rather than this function inventing a +/// reply. `SETNAME`, `SETINFO` and the protocol version write through to the +/// registry, which is what makes them visible to another connection's +/// `CLIENT LIST`. +pub(crate) fn handle_client_command(args: &[String], meta: &mut ClientMeta) -> Value { + let sub = args[0].to_uppercase(); + match (sub.as_str(), args.len()) { + ("ID", 1) => Value::Integer(meta.id as i64), + ("INFO", 1) => Value::BulkString(Some(meta.render().into_bytes())), + ("LIST", 1) => Value::BulkString(Some(client_list_lines().into_bytes())), + ("GETNAME", 1) => { + if meta.name.is_empty() { + Value::BulkString(None) + } else { + Value::BulkString(Some(meta.name.clone().into_bytes())) + } + } + ("SETNAME", 2) => { + // Redis reserves spaces and newlines because the name is echoed + // into the space-separated CLIENT LIST format. + if args[1].contains(' ') || args[1].contains('\n') { + return Value::Error( + "ERR Client names cannot contain spaces, newlines or special characters." + .to_string(), + ); + } + meta.name = args[1].clone(); + publish_client(meta.clone()); + Value::SimpleString("OK".to_string()) + } + ("SETINFO", 3) => match args[1].to_uppercase().as_str() { + "LIB-NAME" => { + meta.lib_name = args[2].clone(); + publish_client(meta.clone()); + Value::SimpleString("OK".to_string()) + } + "LIB-VER" => { + meta.lib_ver = args[2].clone(); + publish_client(meta.clone()); + Value::SimpleString("OK".to_string()) + } + other => Value::Error(format!("ERR Unrecognized option '{other}'")), + }, + ("HELP", 1) => Value::Array(Some( + [ + "CLIENT ", + "ID -- Return this connection's identifier.", + "INFO -- Return information about this connection.", + "LIST -- Return information about all connections.", + "GETNAME -- Return this connection's name.", + "SETNAME -- Set this connection's name.", + "SETINFO -- Identify the client library.", + ] + .iter() + .map(|l| Value::SimpleString(l.to_string())) + .collect(), + )), + // KILL, UNPAUSE, NO-EVICT and friends are administrative operations + // with real semantics. Answering +OK without performing them would be + // worse than saying no: a client would believe a connection had been + // killed or eviction disabled when nothing happened. + _ => unknown_subcommand("CLIENT", &args.join(" ")), + } +} + +/// The configuration parameters `CONFIG GET` reports, resolved from the values +/// actually in force rather than from a table of defaults. +pub(crate) fn config_parameters( + facts: &ServerFacts, + store: &KeyValueStore, +) -> Vec<(&'static str, String)> { + vec![ + ( + "maxmemory", + store.max_memory_bytes().unwrap_or(0).to_string(), + ), + ( + "maxmemory-policy", + eviction_policy_name(store.eviction_policy()).to_string(), + ), + ("maxclients", facts.max_connections.to_string()), + ("port", facts.tcp_port.to_string()), + ( + "tls-port", + if facts.tls_enabled { + facts.tcp_port.to_string() + } else { + "0".to_string() + }, + ), + ( + "appendonly", + if facts.aof_enabled { + "yes".into() + } else { + "no".into() + }, + ), + // Recached has a single keyspace. Clients that SELECT anything other + // than 0 need to know that before they try. + ("databases", "1".to_string()), + // Reported as masked, exactly as Redis does: the presence of a + // password is not a secret, its value is. + ( + "requirepass", + if facts.auth_enabled { + "*".into() + } else { + String::new() + }, + ), + ("proto-max-bulk-len", MAX_BULK_STRING_BYTES.to_string()), + ("timeout", "0".to_string()), + ("save", String::new()), + ] +} + +/// Handle `CONFIG `. +pub(crate) fn handle_config_command( + args: &[String], + facts: &ServerFacts, + store: &KeyValueStore, +) -> Value { + let sub = args[0].to_uppercase(); + match sub.as_str() { + "GET" if args.len() >= 2 => { + let params = config_parameters(facts, store); + let mut out = Vec::new(); + for (name, value) in ¶ms { + if args[1..].iter().any(|pat| glob_match(pat, name)) { + out.push(Value::BulkString(Some(name.as_bytes().to_vec()))); + out.push(Value::BulkString(Some(value.clone().into_bytes()))); + } + } + Value::Array(Some(out)) + } + // Recached reads its configuration from the environment at startup and + // holds it behind an `Arc` for the life of the process, so there is + // nothing a runtime SET could change. Saying so is better than + // returning OK and leaving the operator to discover later that the + // limit they set never applied. + "SET" if args.len() >= 3 => Value::Error(format!( + "ERR CONFIG SET is not supported: Recached is configured at startup. \ + Set '{}' through the environment and restart.", + args[1] + )), + "RESETSTAT" if args.len() == 1 => Value::Error( + "ERR CONFIG RESETSTAT is not supported: counters are exported to Prometheus, \ + where resetting them would break rate calculations." + .to_string(), + ), + _ => unknown_subcommand("CONFIG", &args.join(" ")), + } +} + +/// Handle `CLUSTER `. +/// +/// Recached does not cluster, and this reports that the way Redis does. A +/// `redis-server` that was not started in cluster mode does **not** answer +/// `CLUSTER INFO` with `cluster_enabled:0` — it rejects the whole `CLUSTER` +/// container with this exact sentence, and publishes the flag in `INFO`'s +/// `# Cluster` section instead. Copying the sentence rather than inventing a +/// slot map means a client's "am I clustered" branch takes the same path here +/// as against the server it was written for, and `ERR unknown command` (which +/// is what Recached said before) is the one answer that reads as "too old to +/// ask" rather than "not a cluster". +pub(crate) fn handle_cluster_command(_args: &[String]) -> Value { + Value::Error("ERR This instance has cluster support disabled".to_string()) +} + +/// Handle `MODULE `. +/// +/// There is no module API, so the loaded-module list is empty — which is a +/// real answer, and the same one a stock `redis-server` gives. `LOAD`, +/// `LOADEX` and `UNLOAD` are refused rather than answered `+OK`, because an +/// operator who believes a module loaded has a harder problem than one who +/// was told no. +pub(crate) fn handle_module_command(args: &[String]) -> Value { + match (args[0].to_uppercase().as_str(), args.len()) { + ("LIST", 1) => Value::Array(Some(vec![])), + ("HELP", 1) => Value::Array(Some( + [ + "MODULE ", + "LIST -- Return a list of loaded modules. Recached loads none.", + ] + .iter() + .map(|l| Value::SimpleString((*l).to_string())) + .collect(), + )), + _ => unknown_subcommand("MODULE", &args.join(" ")), + } +} + +/// Handle `PUBSUB [arg ...]` against the live subscriber hub. +/// +/// Recached has shipped `SUBSCRIBE`, `PSUBSCRIBE` and `PUBLISH` from the start +/// with no way to see any of it: `PUBLISH` returns a delivery count, and that +/// was the only observable. The hub already holds both registries, so these +/// three answers are a read of state that existed all along. +/// +/// `SHARDCHANNELS` and `SHARDNUMSUB` are refused rather than answered with the +/// empty array a standalone `redis-server` gives. This is a deliberate +/// divergence: Redis's empty array means "no shard channels are subscribed" on +/// a server where `SSUBSCRIBE` works, and a client reading it would reasonably +/// follow up with one. Recached has no `SSUBSCRIBE` or `SPUBLISH` at all, so +/// the honest answer is that the question does not apply here. +pub(crate) fn handle_pubsub_command(args: &[String], hub: &PubSubHub) -> Value { + match (args[0].to_uppercase().as_str(), args.len()) { + // No pattern means every active channel. Redis matches the pattern + // against channel names with the same globber it uses for keys, and so + // does this — `glob_match` is the one Recached already applies to + // PSUBSCRIBE, so a pattern selects here exactly what it would there. + ("CHANNELS", 1) => Value::Array(Some( + hub.active_channels() + .map(|c| Value::BulkString(Some(c.as_bytes().to_vec()))) + .collect(), + )), + ("CHANNELS", 2) => Value::Array(Some( + hub.active_channels() + .filter(|c| core_engine::store::glob_match(&args[1], c)) + .map(|c| Value::BulkString(Some(c.as_bytes().to_vec()))) + .collect(), + )), + // Flat [channel, count, channel, count, ...]. A channel nobody is + // subscribed to reports 0 rather than being dropped, so a caller that + // asked about N channels can index the reply by position. + ("NUMSUB", _) => { + let mut out = Vec::with_capacity((args.len() - 1) * 2); + for channel in &args[1..] { + out.push(Value::BulkString(Some(channel.as_bytes().to_vec()))); + out.push(Value::Integer(hub.subscriber_count(channel))); + } + Value::Array(Some(out)) + } + ("NUMPAT", 1) => Value::Integer(hub.pattern_count()), + ("HELP", 1) => Value::Array(Some( + [ + "PUBSUB ", + "CHANNELS [pattern] -- Return the currently active channels.", + "NUMSUB [channel ...] -- Return the subscriber count per channel.", + "NUMPAT -- Return the number of distinct subscribed patterns.", + ] + .iter() + .map(|l| Value::SimpleString((*l).to_string())) + .collect(), + )), + _ => unknown_subcommand("PUBSUB", &args.join(" ")), + } +} + +/// Handle `MEMORY ` for everything except `USAGE`, which is a key +/// read and goes to the store. +/// +/// `DOCTOR`, `STATS`, `PURGE` and `MALLOC-STATS` all describe an allocator +/// Recached does not manage — it holds Rust values in a `DashMap` and has no +/// arena to report on or free. Saying so beats a fabricated report. +pub(crate) fn handle_memory_command(args: &[String]) -> Value { + match (args[0].to_uppercase().as_str(), args.len()) { + ("HELP", 1) => Value::Array(Some( + [ + "MEMORY ", + "USAGE [SAMPLES ] -- Bytes held by one key. SAMPLES is accepted \ + and ignored: the estimate always covers every element.", + ] + .iter() + .map(|l| Value::SimpleString((*l).to_string())) + .collect(), + )), + ("DOCTOR" | "STATS" | "PURGE" | "MALLOC-STATS", 1) => Value::Error(format!( + "ERR MEMORY {} is not supported: Recached does not manage its own allocator, \ + so it has nothing to report or free. MEMORY USAGE and INFO memory are the \ + measurements it can make.", + args[0].to_uppercase() + )), + _ => unknown_subcommand("MEMORY", &args.join(" ")), + } +} + +/// `COMMAND INFO`'s per-command reply: name, arity, flags, key positions. +pub(crate) fn command_info_entry(spec: &catalog::CommandSpec) -> Value { + Value::Array(Some(vec![ + Value::BulkString(Some(spec.name.as_bytes().to_vec())), + Value::Integer(spec.arity as i64), + Value::Array(Some( + spec.flags + .iter() + .map(|f| Value::SimpleString((*f).to_string())) + .collect(), + )), + Value::Integer(spec.first_key as i64), + Value::Integer(spec.last_key as i64), + Value::Integer(spec.step as i64), + // ACL categories, tips, key specs and subcommands: Redis 7 appends + // four more elements here. Recached has no ACL system and no + // subcommand tree to describe, so it reports them empty rather than + // omitting them — a client indexing element 6 gets an empty list + // instead of an out-of-range error. + Value::Array(Some(vec![])), + Value::Array(Some(vec![])), + Value::Array(Some(vec![])), + Value::Array(Some(vec![])), + ])) +} + +/// `COMMAND DOCS`'s per-command reply. RESP2 clients see the same pairs as a +/// flat array, which is how Redis degrades a map on the older protocol. +pub(crate) fn command_docs_entry(spec: &catalog::CommandSpec, protover: u8) -> Value { + let fields = vec![ + ( + "summary", + Value::BulkString(Some(spec.summary.as_bytes().to_vec())), + ), + ("since", Value::BulkString(Some(b"1.0.0".to_vec()))), + ( + "group", + Value::BulkString(Some(spec.group.as_bytes().to_vec())), + ), + ("arity", Value::Integer(spec.arity as i64)), + ]; + map_or_flat(fields, protover) +} + +/// RESP3 sends a map; RESP2 has no map type and flattens to alternating +/// key/value entries. Same split `HELLO` already makes. +pub(crate) fn map_or_flat(fields: Vec<(&str, Value)>, protover: u8) -> Value { + if protover >= 3 { + Value::Map( + fields + .into_iter() + .map(|(k, v)| (Value::BulkString(Some(k.as_bytes().to_vec())), v)) + .collect(), + ) + } else { + let mut flat = Vec::with_capacity(fields.len() * 2); + for (k, v) in fields { + flat.push(Value::BulkString(Some(k.as_bytes().to_vec()))); + flat.push(v); + } + Value::Array(Some(flat)) + } +} + +/// Handle `COMMAND [subcommand]`. +pub(crate) fn handle_command_query(args: &[String], protover: u8) -> Value { + let Some(sub) = args.first() else { + // Bare COMMAND: the whole catalog, as COMMAND INFO entries. + return Value::Array(Some( + catalog::CATALOG.iter().map(command_info_entry).collect(), + )); + }; + match sub.to_uppercase().as_str() { + "COUNT" if args.len() == 1 => Value::Integer(catalog::CATALOG.len() as i64), + "LIST" if args.len() == 1 => Value::Array(Some( + catalog::CATALOG + .iter() + .map(|c| Value::BulkString(Some(c.name.as_bytes().to_vec()))) + .collect(), + )), + "INFO" => { + if args.len() == 1 { + return Value::Array(Some( + catalog::CATALOG.iter().map(command_info_entry).collect(), + )); + } + // A name the server does not have replies nil in its slot, so the + // reply stays positionally aligned with the request. + Value::Array(Some( + args[1..] + .iter() + .map(|n| match catalog::lookup(n) { + Some(spec) => command_info_entry(spec), + None => Value::Array(None), + }) + .collect(), + )) + } + "DOCS" => { + let specs: Vec<&catalog::CommandSpec> = if args.len() == 1 { + catalog::CATALOG.iter().collect() + } else { + args[1..] + .iter() + .filter_map(|n| catalog::lookup(n)) + .collect() + }; + // Unknown names are absent from the map rather than nil-filled: + // COMMAND DOCS is keyed by name, so there is no slot to align. + let fields: Vec<(&str, Value)> = specs + .iter() + .map(|s| (s.name, command_docs_entry(s, protover))) + .collect(); + map_or_flat(fields, protover) + } + _ => unknown_subcommand("COMMAND", &args.join(" ")), + } +} + +/// Build the `INFO` payload for `sections` (empty = the default set). +/// +/// The format is load-bearing: `# Section` header, `field:value` lines, CRLF +/// throughout, and a blank line between sections. Every Redis client and +/// monitoring agent parses exactly that shape, so it is covered by tests rather +/// than left to formatting drift. Unknown section names yield no output, which +/// is what Redis does. +#[allow(clippy::too_many_arguments)] +pub(crate) fn render_info( + sections: &[String], + facts: &ServerFacts, + store: &KeyValueStore, + sample: KeyspaceSample, + is_replica: bool, + repl: ReplInfo, + last_save: i64, + live_queries: u64, + watched_keys: u64, +) -> String { + let wanted: Vec<&str> = if sections.is_empty() + || sections + .iter() + .any(|s| s == "all" || s == "everything" || s == "default") + { + DEFAULT_INFO_SECTIONS.to_vec() + } else { + sections.iter().map(|s| s.as_str()).collect() + }; + + let uptime = facts + .start + .elapsed() + .map(|d| d.as_secs()) + .unwrap_or_default(); + let mut out = String::new(); + + for section in wanted { + let body = match section { + "server" => { + format!( + "redis_version:{}\r\n\ + recached_version:{}\r\n\ + redis_mode:standalone\r\n\ + os:{}\r\n\ + arch_bits:{}\r\n\ + process_id:{}\r\n\ + run_id:{}\r\n\ + tcp_port:{}\r\n\ + recached_ws_port:{}\r\n\ + recached_tls_enabled:{}\r\n\ + recached_auth_enabled:{}\r\n\ + uptime_in_seconds:{}\r\n\ + uptime_in_days:{}\r\n", + REDIS_COMPAT_VERSION, + env!("CARGO_PKG_VERSION"), + std::env::consts::OS, + usize::BITS, + std::process::id(), + facts.run_id, + facts.tcp_port, + facts.ws_port, + u8::from(facts.tls_enabled), + u8::from(facts.auth_enabled), + uptime, + uptime / 86_400, + ) + } + "clients" => { + // A negative count would mean the guard accounting is broken; + // clamp rather than emit a value no client can parse. + let active = STAT_CONNECTIONS_ACTIVE.load(Ordering::Relaxed).max(0); + format!( + "connected_clients:{}\r\n\ + maxclients:{}\r\n\ + blocked_clients:0\r\n", + active, facts.max_connections, + ) + } + "memory" => { + let used = sample.memory_bytes as u64; + let max = store.max_memory_bytes().unwrap_or(0) as u64; + format!( + "used_memory:{}\r\n\ + used_memory_human:{}\r\n\ + maxmemory:{}\r\n\ + maxmemory_human:{}\r\n\ + maxmemory_policy:{}\r\n\ + recached_max_keys:{}\r\n", + used, + human_bytes(used), + max, + human_bytes(max), + eviction_policy_name(store.eviction_policy()), + store.max_keys().unwrap_or(0), + ) + } + "persistence" => { + // `loading` is what a client's ready-check reads to decide the + // server can serve traffic. Recached loads its snapshot before + // it binds a listener, so a client that can reach us is never + // looking at a loading server: the answer is always 0. + format!( + "loading:0\r\n\ + rdb_changes_since_last_save:{}\r\n\ + rdb_last_save_time:{}\r\n\ + rdb_bgsave_in_progress:0\r\n\ + aof_enabled:{}\r\n", + store.dirty_count(), + last_save, + u8::from(facts.aof_enabled), + ) + } + "stats" => { + format!( + "total_connections_received:{}\r\n\ + total_commands_processed:{}\r\n\ + keyspace_hits:{}\r\n\ + keyspace_misses:{}\r\n\ + evicted_keys:{}\r\n", + STAT_CONNECTIONS_TOTAL.load(Ordering::Relaxed), + STAT_COMMANDS_TOTAL.load(Ordering::Relaxed), + STAT_KEYSPACE_HITS.load(Ordering::Relaxed), + STAT_KEYSPACE_MISSES.load(Ordering::Relaxed), + store.evicted_count(), + ) + } + "replication" => { + // Redis still spells these `slave`; tooling greps for exactly + // that, so the compatible spelling is authoritative and the + // `replica` names are emitted alongside it. + format!( + "role:{}\r\n\ + connected_slaves:{}\r\n\ + connected_replicas:{}\r\n\ + recached_replication_queue_depth:{}\r\n\ + recached_replication_lag_frames:{}\r\n", + if is_replica { "slave" } else { "master" }, + repl.connected, + repl.connected, + repl.queue_depth, + repl.lag_frames, + ) + } + "keyspace" => { + // Redis omits the db line entirely when the database is empty. + if sample.keys == 0 { + String::new() + } else { + format!( + "db0:keys={},expires={},avg_ttl=0\r\n", + sample.keys, sample.volatile_keys, + ) + } + } + // How a cluster-aware client actually learns it is talking to a + // single node. `CLUSTER INFO` is not that channel: a `redis-server` + // built for standalone answers it with an error, not with + // `cluster_enabled:0`, so this line is the only place the answer + // exists. Reporting it costs one line and stops a client from + // guessing. + "cluster" => "cluster_enabled:0\r\n".to_string(), + // Recached-specific: the live-query machinery has no Redis analogue, + // so it gets its own section rather than being smuggled into one. + "recached" => { + format!( + "live_queries:{}\r\n\ + watched_keys:{}\r\n", + live_queries, watched_keys, + ) + } + _ => continue, + }; + + let title = { + let mut c = section.chars(); + match c.next() { + Some(f) => f.to_uppercase().collect::() + c.as_str(), + None => continue, + } + }; + out.push_str(&format!("# {}\r\n{}\r\n", title, body)); + } + + out +} diff --git a/server-native/src/main.rs b/server-native/src/main.rs index 74cf9b7..97574e7 100644 --- a/server-native/src/main.rs +++ b/server-native/src/main.rs @@ -1,9 +1,69 @@ +//! The Recached server binary: configuration, listeners, and the background +//! loops that outlive any one connection. +//! +//! Everything else lives in a module named for the job it does. Roughly in the +//! order a write travels through them: +//! +//! | Module | Responsibility | +//! |---|---| +//! | [`config`] | `RECACHED_*` parsing, limits, and which values refuse startup | +//! | [`tls`] | Certificate and key loading, for the listener and for replication | +//! | [`connection`] | The RESP and WebSocket command loops, plus HELLO/AUTH | +//! | [`clients`] | Per-connection bookkeeping and the command metrics counters | +//! | [`sync_scopes`] | Signed tokens restricting a socket to a set of key patterns | +//! | [`propagation`] | Turning an executed command into the frame everyone else replays | +//! | [`server_state`] | Shared state a write passes through: AOF, replicas, role | +//! | [`persistence`] | Snapshots and the append-only file | +//! | [`replication`] | Serving replicas, and following a primary | +//! | [`pubsub`] | The channel/pattern subscriber hub | +//! | [`watch`] | WATCH's compare-and-set registry and live queries | +//! | [`mod@info`] | INFO, CLIENT, CONFIG, COMMAND — reporting on the server, not the data | +//! +//! Modules pull crate-wide names in with `use crate::*`, and this file re-globs +//! each of them, so a shared type is nameable everywhere without a bespoke +//! import list per file. Items shared across modules are `pub(crate)`; nothing +//! here is a public API. +//! +//! **The one duplication worth knowing about:** [`connection`] carries two +//! near-identical command loops, one per transport. They have drifted before — +//! a transaction bug once had to be fixed twice — so a change to one almost +//! always belongs in the other. + // jemalloc isn't available under MSVC (see Cargo.toml); fall back to the // system allocator there. #[cfg(not(target_env = "msvc"))] #[global_allocator] static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; +mod clients; +mod config; +mod connection; +mod info; +mod persistence; +mod propagation; +mod pubsub; +mod replication; +mod server_state; +mod sync_scopes; +mod tls; +mod watch; + +#[cfg(test)] +mod tests; + +use clients::*; +use config::*; +use connection::*; +use info::*; +use persistence::*; +use propagation::*; +use pubsub::*; +use replication::*; +use server_state::*; +use sync_scopes::*; +use tls::*; +use watch::*; + use core_engine::catalog; use core_engine::cmd::{Command, SetExpiry, ZAddCondition}; use core_engine::resp::{MAX_BULK_STRING_BYTES, Value}; @@ -37,11263 +97,787 @@ use tracing::{debug, error, info, warn}; // ── metrics ─────────────────────────────────────────────────────────────────── -/// Counters mirrored out of the `metrics` registry so `INFO` can read them. -/// -/// `metrics::Counter` and `Gauge` handles are write-only — there is no way to -/// read a recorded value back out — so every number `INFO` reports from the -/// registry needs a plain atomic alongside it. These are the only ones INFO -/// needs; the rest of its fields come from the store or `ServerState`. -static STAT_CONNECTIONS_TOTAL: AtomicU64 = AtomicU64::new(0); -static STAT_CONNECTIONS_ACTIVE: AtomicI64 = AtomicI64::new(0); -static STAT_COMMANDS_TOTAL: AtomicU64 = AtomicU64::new(0); -static STAT_KEYSPACE_HITS: AtomicU64 = AtomicU64::new(0); -static STAT_KEYSPACE_MISSES: AtomicU64 = AtomicU64::new(0); - -/// RAII guard that tracks an active connection. Increments on creation, -/// decrements when dropped (i.e. when the handler future completes), and -/// keeps the `CLIENT LIST` registry in step with both. -struct ConnectionGuard { - id: u64, -} - -impl ConnectionGuard { - fn new(kind: &'static str, meta: ClientMeta) -> Self { - counter!("recached_connections_total", "type" => kind).increment(1); - gauge!("recached_connections_active").increment(1.0); - STAT_CONNECTIONS_TOTAL.fetch_add(1, Ordering::Relaxed); - STAT_CONNECTIONS_ACTIVE.fetch_add(1, Ordering::Relaxed); - let id = meta.id; - publish_client(meta); - Self { id } - } -} - -impl Drop for ConnectionGuard { - fn drop(&mut self) { - gauge!("recached_connections_active").decrement(1.0); - STAT_CONNECTIONS_ACTIVE.fetch_sub(1, Ordering::Relaxed); - CLIENTS - .write() - .unwrap_or_else(|e| e.into_inner()) - .remove(&self.id); - } +fn now_unix_secs() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs() as i64 } -// ── Client registry ─────────────────────────────────────────────────────────── - -/// What `CLIENT INFO` and `CLIENT LIST` report about one live connection. +/// Wall-clock milliseconds since the epoch, the unit every stored expiry uses. /// -/// The connection task owns the authoritative copy and republishes it whenever -/// a field changes — a name is set, a library identifies itself, the protocol -/// is renegotiated. Publishing on change rather than per command keeps the -/// registry's write lock off the hot path: these events happen a handful of -/// times per connection, commands happen millions of times. -#[derive(Clone, Debug)] -struct ClientMeta { - id: u64, - /// Peer address, or empty when the listener could not report one. - addr: String, - laddr: String, - name: String, - lib_name: String, - lib_ver: String, - since: SystemTime, - resp: u8, - sub: usize, - psub: usize, +/// Read once per propagated write so [`broadcast_for`] can turn a *relative* +/// TTL into an absolute deadline — see the comment there for why that matters. +fn now_unix_ms() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() as u64 } -impl ClientMeta { - fn new(id: u64, addr: String, laddr: String) -> Self { - Self { - id, - addr, - laddr, - name: String::new(), - lib_name: String::new(), - lib_ver: String::new(), - since: SystemTime::now(), - resp: 2, - sub: 0, - psub: 0, - } +#[tokio::main] +async fn main() -> Result<(), Box> { + // All runtime configuration is via RECACHED_* env vars; the only flags are + // --version/-V (required by e.g. the Homebrew formula's install test). + if std::env::args().any(|a| a == "--version" || a == "-V") { + println!("recached-server {}", env!("CARGO_PKG_VERSION")); + return Ok(()); } - /// One line in Redis's `CLIENT LIST` format: space-separated `key=value`. - /// - /// Only fields Recached can answer truthfully are emitted. Redis also - /// reports buffer sizes, file descriptors and an event mask; inventing - /// plausible numbers for those would be worse than leaving them out, - /// because a client cannot tell a made-up `omem` from a real one. Parsers - /// read this format key by key and skip what they do not recognise, so a - /// shorter line is a supported line. - fn render(&self) -> String { - let age = self.since.elapsed().unwrap_or_default().as_secs(); - format!( - "id={} addr={} laddr={} name={} age={} idle=0 flags=N db=0 \ - sub={} psub={} multi=-1 resp={} lib-name={} lib-ver={}", - self.id, - self.addr, - self.laddr, - self.name, - age, - self.sub, - self.psub, - self.resp, - self.lib_name, - self.lib_ver, + tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")), ) - } -} - -/// Live connections, keyed by id. A `BTreeMap` so `CLIENT LIST` comes out in -/// connection order rather than an arbitrary one that shuffles between calls. -static CLIENTS: std::sync::LazyLock>> = - std::sync::LazyLock::new(Default::default); - -fn publish_client(meta: ClientMeta) { - CLIENTS - .write() - .unwrap_or_else(|e| e.into_inner()) - .insert(meta.id, meta); -} - -fn client_list_lines() -> String { - let clients = CLIENTS.read().unwrap_or_else(|e| e.into_inner()); - let mut out = String::new(); - for meta in clients.values() { - out.push_str(&meta.render()); - out.push('\n'); - } - out -} - -fn command_name(cmd: &Command) -> &'static str { - match cmd { - Command::Ping(_) => "ping", - Command::Auth(_) => "auth", - Command::Hello(_) => "hello", - Command::Info(_) => "info", - Command::Quit => "quit", - Command::Client(_) => "client", - Command::Config(_) => "config", - Command::CommandQuery(_) => "command", - Command::Cluster(_) => "cluster", - Command::Module(_) => "module", - Command::PubSub(_) => "pubsub", - Command::Memory(_) | Command::MemoryUsage(_) => "memory", - Command::Get(_) => "get", - Command::ESet(_, _) => "eset", - Command::Set(_, _, _) => "set", - Command::Del(_) => "del", - Command::Unlink(_) => "unlink", - Command::Append(_, _) => "append", - Command::Strlen(_) => "strlen", - Command::GetRange(_, _, _) => "getrange", - Command::GetSet(_, _) => "getset", - Command::MGet(_) => "mget", - Command::MSet(_) => "mset", - Command::SetNx(_, _) => "setnx", - Command::SetEx(_, _, _) => "setex", - Command::PSetEx(_, _, _) => "psetex", - Command::Incr(_) => "incr", - Command::Decr(_) => "decr", - Command::IncrBy(_, _) => "incrby", - Command::DecrBy(_, _) => "decrby", - Command::Expire(_, _) => "expire", - Command::PExpire(_, _) => "pexpire", - Command::ExpireAt(_, _) => "expireat", - Command::PExpireAt(_, _) => "pexpireat", - Command::Ttl(_) => "ttl", - Command::PTtl(_) => "pttl", - Command::Persist(_) => "persist", - Command::Exists(_) => "exists", - Command::Keys(_) => "keys", - Command::Scan(_, _, _) => "scan", - Command::DbSize => "dbsize", - Command::FlushDb => "flushdb", - Command::Rename(_, _) => "rename", - Command::Type(_) => "type", - Command::HSet(_, _) => "hset", - Command::HGet(_, _) => "hget", - Command::HGetAll(_) => "hgetall", - Command::HDel(_, _) => "hdel", - Command::HKeys(_) => "hkeys", - Command::HVals(_) => "hvals", - Command::HLen(_) => "hlen", - Command::HIncrBy(_, _, _) => "hincrby", - Command::HIncrByFloat(_, _, _) => "hincrbyfloat", - Command::HExists(_, _) => "hexists", - Command::HSetNx(_, _, _) => "hsetnx", - Command::HMGet(_, _) => "hmget", - Command::HScan(_, _) => "hscan", - Command::LPush(_, _) => "lpush", - Command::RPush(_, _) => "rpush", - Command::LPushX(_, _) => "lpushx", - Command::RPushX(_, _) => "rpushx", - Command::LPop(_, _) => "lpop", - Command::RPop(_, _) => "rpop", - Command::LRange(_, _, _) => "lrange", - Command::LLen(_) => "llen", - Command::LIndex(_, _) => "lindex", - Command::LSet(_, _, _) => "lset", - Command::LRem(_, _, _) => "lrem", - Command::LTrim(_, _, _) => "ltrim", - Command::SAdd(_, _) => "sadd", - Command::SMembers(_) => "smembers", - Command::SRem(_, _) => "srem", - Command::SCard(_) => "scard", - Command::SIsMember(_, _) => "sismember", - Command::SMIsMember(_, _) => "smismember", - Command::SInter(_) => "sinter", - Command::SInterStore(_, _) => "sinterstore", - Command::SUnion(_) => "sunion", - Command::SUnionStore(_, _) => "sunionstore", - Command::SDiff(_) => "sdiff", - Command::SDiffStore(_, _) => "sdiffstore", - Command::SPop(_, _) => "spop", - Command::SRandMember(_, _) => "srandmember", - Command::SMove(_, _, _) => "smove", - Command::SScan(_, _) => "sscan", - Command::ZAdd(_, _, _) => "zadd", - Command::ZRange(_, _, _, _) => "zrange", - Command::ZRevRange(_, _, _, _) => "zrevrange", - Command::ZRangeByScore(_, _, _, _, _) => "zrangebyscore", - Command::ZRevRangeByScore(_, _, _, _, _) => "zrevrangebyscore", - Command::ZScore(_, _) => "zscore", - Command::ZMScore(_, _) => "zmscore", - Command::ZRank(_, _) => "zrank", - Command::ZRevRank(_, _) => "zrevrank", - Command::ZRem(_, _) => "zrem", - Command::ZCard(_) => "zcard", - Command::ZIncrBy(_, _, _) => "zincrby", - Command::ZCount(_, _, _) => "zcount", - Command::ZScan(_, _) => "zscan", - Command::Multi => "multi", - Command::Exec => "exec", - Command::Discard => "discard", - Command::Subscribe(_) => "subscribe", - Command::Unsubscribe(_) => "unsubscribe", - Command::PSubscribe(_) => "psubscribe", - Command::PUnsubscribe(_) => "punsubscribe", - Command::Publish(_, _) => "publish", - Command::Watch(_) => "watch", - Command::Unwatch(_) => "unwatch", - Command::Save => "save", - Command::BgSave => "bgsave", - Command::LastSave => "lastsave", - Command::ReplicaOfNoOne => "replicaof", - Command::JSet(_, _, _) => "jset", - Command::JGet(_, _) => "jget", - Command::JMerge(_, _) => "jmerge", - Command::RlSet(_, _, _) => "rlset", - Command::RlCheck(_, _) => "rlcheck", - Command::Sync(_) => "sync", - Command::QSub(_) => "qsub", - Command::QUnsub(_) => "qunsub", - // Metrics count the wrapped command, not the envelope. - Command::Dedup(_, _, inner) => command_name(inner), - Command::Unknown(_) => UNKNOWN_COMMAND, - } -} - -/// Per-command counter handles, resolved through the metrics registry once and -/// then reused — the registry lookup (key construction + shard lock) is too -/// expensive to pay on every command. Keyed by the `&'static str` from -/// `command_name`. -/// -/// Built in one shot from the command catalog on first use, which is after -/// `main` has installed the recorder, and never written again. It used to be an -/// `RwLock` filled in as each command was first seen, which cost a -/// read-lock acquisition on *every command on every connection* — one -/// contended atomic in the hot path of a server whose whole job is throughput — -/// and `.unwrap()`ed the lock, so a single panic anywhere holding it would -/// poison the lock and make every subsequent command panic for the life of the -/// process. An immutable map needs no lock and cannot be poisoned. -static CMD_COUNTERS: std::sync::LazyLock> = - std::sync::LazyLock::new(|| { - core_engine::catalog::CATALOG - .iter() - .map(|spec| { - ( - spec.name, - counter!("recached_commands_total", "command" => spec.name), - ) - }) - .chain(std::iter::once(( - UNKNOWN_COMMAND, - counter!("recached_commands_total", "command" => UNKNOWN_COMMAND), - ))) - .collect() - }); - -/// Label used for a command the parser did not recognise. Not a catalog row, so -/// it is registered alongside them. -const UNKNOWN_COMMAND: &str = "unknown"; - -fn record_command(name: &'static str) { - STAT_COMMANDS_TOTAL.fetch_add(1, Ordering::Relaxed); - match CMD_COUNTERS.get(name) { - Some(c) => c.increment(1), - // A name `command_name` can produce but the catalog does not list. - // `command_name_labels_are_all_pre_registered` fails CI if that ever - // happens, so this is a correctness backstop, not a routine path. - None => counter!("recached_commands_total", "command" => name).increment(1), - } -} - -static KEYSPACE_HITS: std::sync::LazyLock = - std::sync::LazyLock::new(|| counter!("recached_keyspace_hits_total")); -static KEYSPACE_MISSES: std::sync::LazyLock = - std::sync::LazyLock::new(|| counter!("recached_keyspace_misses_total")); - -/// Executes `cmd`, recording metrics and the dirty counter. Takes the command -/// by value — the hot path hands it straight to the store without a clone; -/// callers that still need the command afterwards (write fan-out) clone first. -fn execute_and_record(store: &KeyValueStore, cmd: Command) -> Value { - let name = command_name(&cmd); - let is_write = is_write_command(&cmd); - let is_get = matches!(cmd, Command::Get(_)); - let response = store.execute(cmd); - record_command(name); - if matches!(response, Value::Error(_)) { - counter!("recached_command_errors_total", "command" => name).increment(1); - } else if is_write { - store.mark_dirty(); - } - if is_get { - match &response { - Value::BulkString(Some(_)) => { - KEYSPACE_HITS.increment(1); - STAT_KEYSPACE_HITS.fetch_add(1, Ordering::Relaxed); - } - Value::BulkString(None) => { - KEYSPACE_MISSES.increment(1); - STAT_KEYSPACE_MISSES.fetch_add(1, Ordering::Relaxed); - } - _ => {} - } - } - response -} - -/// True when at least one consumer of write effects exists (WebSocket peers, -/// AOF, replicas, or watched keys). When false — the common standalone case — -/// the caller can move the command into `execute_and_record` without cloning -/// and skip `apply_write_effects` entirely. -fn write_effects_armed( - tx: &broadcast::Sender, - state: &ServerState, - watch_registry: &WatchRegistry, -) -> bool { - tx.receiver_count() > 0 || state.needs_write_log() || !watch_registry.is_empty() -} - -// ── TCP listeners ───────────────────────────────────────────────────────────── + .init(); -/// Binds `n` TCP sockets on `addr`, all with `SO_REUSEPORT`, so the OS can -/// distribute incoming connections across multiple accept loops — one per -/// Tokio worker thread. Falls back to a single plain `TcpListener::bind` on -/// platforms that don't support `SO_REUSEPORT`. -fn make_tcp_listeners(addr: &str, n: usize) -> std::io::Result> { - use socket2::{Domain, Socket, Type}; - let socket_addr: std::net::SocketAddr = addr - .parse() - .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidInput, e))?; - let domain = if socket_addr.is_ipv6() { - Domain::IPV6 + // ── bind address ────────────────────────────────────────────────────── + // Host/interface all listeners bind to. Defaults to 0.0.0.0 (all + // interfaces) for backwards compatibility; set RECACHED_BIND=127.0.0.1 to + // restrict to localhost, which — together with RECACHED_PASSWORD — is + // strongly recommended unless the server is deliberately public. + let bind_host = std::env::var("RECACHED_BIND").unwrap_or_else(|_| "0.0.0.0".to_string()); + if bind_host == "0.0.0.0" { + warn!( + "Binding all interfaces (0.0.0.0). Set RECACHED_BIND=127.0.0.1 and RECACHED_PASSWORD before exposing this host." + ); } else { - Domain::IPV4 - }; - // SO_REUSEPORT — which lets multiple sockets share one port — is Unix-only. - // Without it, binding a second socket to the same port fails, so fall back - // to a single accept loop on non-Unix platforms. - #[cfg(not(unix))] - let n = 1; - let mut out = Vec::with_capacity(n); - for _ in 0..n { - let sock = Socket::new(domain, Type::STREAM, None)?; - sock.set_reuse_address(true)?; - #[cfg(unix)] - sock.set_reuse_port(true)?; - sock.set_nonblocking(true)?; - sock.bind(&socket_addr.into())?; - sock.listen(4096)?; - let std_listener: std::net::TcpListener = sock.into(); - out.push(TcpListener::from_std(std_listener)?); - } - Ok(out) -} - -// ── TLS ─────────────────────────────────────────────────────────────────────── - -// PEM parsing comes from rustls-pki-types, the crate rustls itself uses. -// `rustls-pemfile` was deprecated in favour of it (RUSTSEC-2025-0134), and an -// unmaintained dependency is a poor thing to have sitting in the TLS path. -fn load_certs(path: &str) -> std::io::Result>> { - CertificateDer::pem_file_iter(path) - .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string()))? - .map(|r| r.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string()))) - .collect() -} - -fn load_private_key(path: &str) -> std::io::Result> { - PrivateKeyDer::from_pem_file(path) - .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string())) -} - -/// Decides what a (cert, key) environment pair means, without touching the -/// filesystem — split out from `load_tls_acceptor` so the security-relevant -/// rule is unit-testable. -/// -/// Setting only one half is treated as a fatal misconfiguration rather than a -/// fallback to plaintext: an operator who set `RECACHED_TLS_CERT` intends TLS, -/// and silently serving unencrypted traffic on both ports because the key -/// variable was misspelled is a failure they would not detect until traffic had -/// already been exposed. -fn resolve_tls_paths( - cert: Option, - key: Option, -) -> Result, String> { - match (cert, key) { - (None, None) => Ok(None), - (Some(c), Some(k)) => Ok(Some((c, k))), - (Some(_), None) => Err( - "RECACHED_TLS_CERT is set but RECACHED_TLS_KEY is not — refusing to start rather than \ - silently serving plaintext. Set both, or neither." - .to_string(), - ), - (None, Some(_)) => Err( - "RECACHED_TLS_KEY is set but RECACHED_TLS_CERT is not — refusing to start rather than \ - silently serving plaintext. Set both, or neither." - .to_string(), - ), - } -} - -/// The name a replica verifies the primary's certificate against. -/// -/// Defaults to the host portion of `RECACHED_REPLICAOF`, which is what an -/// operator means by "connect to this primary". It is overridable because the -/// address is frequently an IP while the certificate names a host: a cert issued -/// for `primary.internal` does not validate against `10.0.1.5` unless it also -/// carries that IP as a SAN, and pointing at the IP is the common deployment. -fn repl_tls_servername(primary_addr: &str, override_name: Option) -> String { - if let Some(name) = override_name.map(|n| n.trim().to_string()) - && !name.is_empty() - { - return name; - } - // `host:port`, or a bare host. An IPv6 literal is bracketed, so splitting on - // the last colon would cut inside the address. - match primary_addr.rsplit_once(':') { - Some((host, _)) if !host.is_empty() && !host.contains(':') => host.to_string(), - _ => primary_addr - .trim_start_matches('[') - .split(']') - .next() - .unwrap_or(primary_addr) - .to_string(), - } -} - -/// Build the TLS connector a replica uses to reach its primary. -/// -/// The trust anchor is an explicit file rather than the system root store, and -/// that is deliberate. Replication is a link between two machines the same -/// operator runs, so the right model is pinning the certificate (or the private -/// CA that issued it) — not trusting every public CA on earth to vouch for a -/// host that streams the entire keyspace. Pointing this at a system bundle still -/// works if the primary genuinely uses a publicly-issued certificate. -fn load_repl_tls_connector(ca_path: &str) -> Result { - let certs = - load_certs(ca_path).map_err(|e| format!("RECACHED_REPL_TLS_CA '{ca_path}': {e}"))?; - if certs.is_empty() { - return Err(format!( - "RECACHED_REPL_TLS_CA '{ca_path}' contains no certificates — replication TLS would \ - trust nothing and every connection would fail." - )); - } - let mut roots = RootCertStore::empty(); - for cert in certs { - roots - .add(cert) - .map_err(|e| format!("RECACHED_REPL_TLS_CA '{ca_path}': {e}"))?; + info!("Binding interface {}", bind_host); } - let config = ClientConfig::builder() - .with_root_certificates(roots) - .with_no_client_auth(); - Ok(TlsConnector::from(Arc::new(config))) -} -/// Returns a `TlsAcceptor` when both `RECACHED_TLS_CERT` and `RECACHED_TLS_KEY` -/// are set, `None` when neither is. Exits if exactly one is set. -fn load_tls_acceptor() -> Option { - let (cert_path, key_path) = match resolve_tls_paths( - std::env::var("RECACHED_TLS_CERT").ok(), - std::env::var("RECACHED_TLS_KEY").ok(), + // ── Listening ports ─────────────────────────────────────────────────── + // Defaults are Redis's 6379 and Recached's 6380, so an existing deployment + // needs no configuration. They are overridable because they were not: the + // two ports were compiled in, which made a second instance on one host + // impossible — including a replica alongside its primary — and left no way + // to move off 6379, the port every commodity scanner probes first. + let (tcp_port, ws_port) = match ( + parse_env_port("RECACHED_PORT", 6379), + parse_env_port("RECACHED_WS_PORT", 6380), ) { - Ok(None) => return None, - Ok(Some(pair)) => pair, - Err(msg) => { + (Ok(t), Ok(w)) if t == w => { + error!("RECACHED_PORT and RECACHED_WS_PORT are both {t}; they must differ."); + std::process::exit(1); + } + (Ok(t), Ok(w)) => (t, w), + (Err(msg), _) | (_, Err(msg)) => { error!("{msg}"); std::process::exit(1); } }; - let cert_coll = load_certs(&cert_path).unwrap_or_else(|e| panic!("TLS cert {cert_path}: {e}")); - let key = load_private_key(&key_path).unwrap_or_else(|e| panic!("TLS key {key_path}: {e}")); - - let config = ServerConfig::builder() - .with_no_client_auth() - .with_single_cert(cert_coll, key) - .expect("invalid TLS configuration"); - - Some(TlsAcceptor::from(Arc::new(config))) -} - -// ── tunables ──────────────────────────────────────────────────────────────── - -const TCP_READ_BUFFER_BYTES: usize = 16 * 1024; // 16 KB — matches Redis default -const MAX_TCP_READ_BUFFER_BYTES: usize = 64 * 1024 * 1024; // 64 MB per connection -/// Per-connection limits. Compiled-in defaults, overridable at startup because -/// the right value is workload-dependent — Redis exposes `maxmemory-samples` -/// for the same reason. Read once and cached; changing one needs a restart. -fn env_limit(var: &str, default: usize) -> usize { - std::env::var(var) - .ok() - .and_then(|v| v.trim().parse::().ok()) - .filter(|n| *n > 0) - .unwrap_or(default) -} - -/// Commands queued inside one `MULTI`. Override: `RECACHED_MAX_MULTI_QUEUE`. -fn max_multi_queue_len() -> usize { - static V: std::sync::OnceLock = std::sync::OnceLock::new(); - *V.get_or_init(|| env_limit("RECACHED_MAX_MULTI_QUEUE", 10_000)) -} - -/// Keys one connection may `WATCH`. Override: `RECACHED_MAX_WATCHES_PER_CONN`. -fn max_watches_per_conn() -> usize { - static V: std::sync::OnceLock = std::sync::OnceLock::new(); - *V.get_or_init(|| env_limit("RECACHED_MAX_WATCHES_PER_CONN", 1_024)) -} - -/// Live queries one connection may hold. Override: `RECACHED_MAX_LIVE_QUERIES`. -fn max_qsubs_per_conn() -> usize { - static V: std::sync::OnceLock = std::sync::OnceLock::new(); - *V.get_or_init(|| env_limit("RECACHED_MAX_LIVE_QUERIES", 64)) -} -/// Cap on the number of key/value pairs returned as QSUB initial state, so a -/// pattern matching a huge keyspace cannot produce an unbounded reply frame. -/// Keys returned in a live query's initial state. -/// Override: `RECACHED_MAX_QSUB_INITIAL_KEYS`. -fn max_qsub_initial_keys() -> usize { - static V: std::sync::OnceLock = std::sync::OnceLock::new(); - *V.get_or_init(|| env_limit("RECACHED_MAX_QSUB_INITIAL_KEYS", 10_000)) -} -const BROADCAST_CHANNEL_CAPACITY: usize = 512; -const DEFAULT_MAX_CONNECTIONS: usize = 1024; -const MAX_AUTH_FAILURES: u32 = 5; -const EVICTION_INTERVAL_SECS: u64 = 1; -const DEFAULT_HANDSHAKE_TIMEOUT_SECS: u64 = 10; -/// Longest replication auth line accepted, in bytes. -const MAX_REPL_AUTH_LINE: usize = 512; -/// Window over which failed replication auth attempts are counted per peer. -const REPL_AUTH_WINDOW: Duration = Duration::from_secs(60); -/// Peers tracked before the throttle sweeps expired entries. -const REPL_AUTH_SWEEP_THRESHOLD: usize = 1024; - -// ── private file writes ─────────────────────────────────────────────────────── - -/// Write `bytes` to `path`, creating it readable only by this user. -/// -/// Snapshots, the AOF, and the dedup sidecar are plaintext MessagePack dumps of -/// the keyspace. `fs::write` creates with the process umask — `0644` on a -/// typical host — so any local user could read the entire cache. The -/// documentation told operators to protect these files with filesystem -/// permissions; the server should never have been relying on that. -/// -/// Permissions are also set explicitly after opening, so a file left behind -/// `0644` by an earlier version is tightened on the next write rather than -/// keeping its old mode forever. -/// Writes are fsynced before returning. Every caller is writing state that has -/// to survive a crash — a snapshot about to be renamed into place, or the dedup -/// high-water marks that stop a replayed write being applied twice — so the -/// barrier belongs here rather than at each call site. -async fn write_private(path: &std::path::Path, bytes: &[u8]) -> std::io::Result<()> { - let mut opts = tokio::fs::OpenOptions::new(); - opts.write(true).create(true).truncate(true); - #[cfg(unix)] - opts.mode(0o600); - let mut f = opts.open(path).await?; - #[cfg(unix)] - restrict_permissions(&f).await; - f.write_all(bytes).await?; - f.flush().await?; - // `sync_all`, not `sync_data`: this file was just created, so its metadata - // is part of what has to reach the device. - f.sync_all().await?; - Ok(()) -} - -/// fsync the directory holding `path`, making a `rename` into it durable. -/// -/// Renaming a fsynced temp file over the target is atomic with respect to -/// readers, but the *directory entry* is itself just a write: without this, a -/// crash can leave the old file, or no file, despite the new contents being -/// safely on disk. Only meaningful on unix — Windows has no directory handle to -/// sync — so the call is compiled out elsewhere. -#[cfg(unix)] -async fn sync_parent_dir(path: &std::path::Path) { - let Some(dir) = path.parent() else { - return; - }; - // An empty parent means the path was relative with no directory component. - let dir = if dir.as_os_str().is_empty() { - std::path::Path::new(".") - } else { - dir - }; - match tokio::fs::File::open(dir).await { - Ok(f) => { - if let Err(e) = f.sync_all().await { - warn!("Directory fsync failed for {:?}: {}", dir, e); - } - } - Err(e) => warn!("Could not open {:?} to fsync: {}", dir, e), - } -} - -#[cfg(not(unix))] -async fn sync_parent_dir(_path: &std::path::Path) {} - -/// Tighten an already-open file to `0600`, ignoring failure. -/// -/// Best-effort by design: on a filesystem that cannot represent unix modes this -/// is not something to fail a write over, and the caller has already created the -/// file with the right mode where the platform allows it. -#[cfg(unix)] -async fn restrict_permissions(f: &tokio::fs::File) { - use std::os::unix::fs::PermissionsExt; - let _ = f - .set_permissions(std::fs::Permissions::from_mode(0o600)) - .await; -} - -/// Path for a temp file alongside `path`, distinct per process. -/// -/// The previous fixed `.tmp` name meant two servers sharing a directory would -/// clobber each other's half-written snapshot, and made the target predictable -/// to anyone who could already write to that directory. Residual: this is not -/// unguessable, so it is a defence against collision rather than against an -/// attacker who already controls the data directory. -fn temp_sibling(path: &std::path::Path, tag: &str) -> PathBuf { - path.with_extension(format!("{tag}.{}.tmp", std::process::id())) -} - -// ── snapshot persistence ────────────────────────────────────────────────────── - -struct SnapshotConfig { - path: PathBuf, - last_save: AtomicI64, -} - -fn now_unix_secs() -> i64 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_secs() as i64 -} - -/// Parse a human-readable memory size string (e.g. "512mb", "1gb", "262144") -/// into a byte count. Returns None on parse failure. -/// Parse `RECACHED_ALLOW_IPS` into exact addresses. -/// -/// Every entry must parse. The previous behaviour logged and dropped invalid -/// entries, which quietly narrowed a security control: a mistyped CIDR range -/// like `10.0.0.0/8` produced an allowlist that did not include the hosts the -/// operator wrote, and an entirely invalid list produced an empty one — which -/// rejects *every* connection while the server still reports itself healthy. -/// Refusing to start makes the misconfiguration impossible to miss. -fn parse_allow_ips(raw: &str) -> Result, String> { - let mut ips = Vec::new(); - for entry in raw.split(',') { - let trimmed = entry.trim(); - if trimmed.is_empty() { - continue; + // ── Prometheus metrics ──────────────────────────────────────────────── + // `0` means "do not export", which the reference has always documented and + // the code never honoured: `0` parsed fine, and binding `host:0` hands the + // listener an OS-assigned ephemeral port. An operator who set it to turn + // metrics *off* got them served on an unpredictable port instead — the + // opposite of what they asked for, and unlikely to be noticed. + let metrics_port = match parse_env_metrics_port() { + Ok(p) => p, + Err(msg) => { + error!("{msg}"); + std::process::exit(1); } - match IpAddr::from_str(trimmed) { - Ok(ip) => ips.push(ip), - Err(_) => { - return Err(format!( - "RECACHED_ALLOW_IPS: '{trimmed}' is not a valid IP address. Exact addresses \ - only — CIDR ranges and hostnames are not supported. Refusing to start rather \ - than applying a narrower allowlist than configured." - )); + }; + match metrics_port { + None => info!("Prometheus metrics DISABLED (RECACHED_METRICS_PORT=0)."), + Some(port) => { + let metrics_addr: std::net::SocketAddr = match format!("{}:{}", bind_host, port).parse() + { + Ok(a) => a, + Err(_) => { + error!( + "RECACHED_BIND '{bind_host}' and RECACHED_METRICS_PORT {port} do not \ + form a valid address. An IPv6 bind host must be bracketed, as in \ + [::1]." + ); + std::process::exit(1); + } + }; + if let Err(e) = metrics_exporter_prometheus::PrometheusBuilder::new() + .with_http_listener(metrics_addr) + .install() + { + // Almost always a second instance on the same host: the data + // ports are configurable, so this one has to be too, or the + // collision simply moves here — and it used to arrive as a + // panic with a backtrace note, which reads like a bug in + // Recached rather than two servers wanting one port. + error!( + "Could not start the Prometheus exporter on {metrics_addr}: {e}. \ + If another Recached instance is already running on this host, give this one \ + its own RECACHED_METRICS_PORT (as well as RECACHED_PORT and \ + RECACHED_WS_PORT), or set RECACHED_METRICS_PORT=0 to disable metrics." + ); + std::process::exit(1); } + info!("Prometheus metrics at http://{}/metrics", metrics_addr); } } - if ips.is_empty() { - return Err( - "RECACHED_ALLOW_IPS is set but contains no valid addresses — this would reject every \ - connection. Unset it to accept all connections." - .to_string(), - ); - } - Ok(ips) -} - -/// Parse a boolean environment variable, rejecting anything ambiguous. -/// -/// Silently treating `RECACHED_REPL_ENABLE=please` as false would leave an -/// operator believing replication was on when it was not; treating it as true -/// would open a port they never asked for. Neither is acceptable for a variable -/// that gates a security boundary, so an unrecognised value refuses to start. -fn parse_env_bool(var: &str, raw: &str) -> Result { - match raw.trim().to_ascii_lowercase().as_str() { - "1" | "true" | "yes" | "on" => Ok(true), - "0" | "false" | "no" | "off" => Ok(false), - other => Err(format!( - "{var}: '{other}' is not a boolean. Use 1/true/yes/on or 0/false/no/off." - )), - } -} -/// True when `bind_host` can only be reached from this machine. -/// -/// A hostname that does not parse as an address is treated as public: the -/// conservative answer is the one that demands a password. -fn bind_is_loopback(bind_host: &str) -> bool { - if bind_host.eq_ignore_ascii_case("localhost") { - return true; - } - // An IPv6 bind address has to be written bracketed — `[::1]` — because the - // listeners format it as `{host}:{port}`, and `::1:6379` does not parse. - // Without stripping them, `[::1]` fails to parse as an address and would be - // classified as public, demanding a replication password on what is in fact - // loopback. - let host = bind_host - .strip_prefix('[') - .and_then(|h| h.strip_suffix(']')) - .unwrap_or(bind_host); - IpAddr::from_str(host) - .map(|ip| ip.is_loopback()) - .unwrap_or(false) -} + // ── auth ────────────────────────────────────────────────────────────── + let password = std::env::var("RECACHED_PASSWORD").ok(); + let global_password = Arc::new(password); -/// Whether to bind the replication listener, given the environment. -/// -/// This listener used to start unconditionally on every node. The stated reason -/// was multi-tier replication — a replica must be able to serve sub-replicas — -/// but the cost was that *every* default deployment opened a port carrying the -/// entire keyspace to anyone who connected: with no `RECACHED_REPL_PASSWORD`, -/// `handle_replica` skips the handshake and sends a full snapshot followed by a -/// live stream of every subsequent write. An operator who set -/// `RECACHED_PASSWORD` had every reason to believe the data was behind -/// authentication, and it was not. -/// -/// Two rules now apply. The port is closed unless `RECACHED_REPL_ENABLE` says -/// otherwise, and enabling it on an interface other than loopback without a -/// password refuses to start rather than serving the keyspace unauthenticated. -/// Multi-tier replication still works — a node that serves sub-replicas sets -/// the variable, which is the point: it is now a decision rather than a default. -fn resolve_repl_listen( - enable: Option, - bind_host: &str, - password: Option<&str>, -) -> Result { - let enabled = match enable.as_deref().map(str::trim) { - None | Some("") => false, - Some(v) => parse_env_bool("RECACHED_REPL_ENABLE", v)?, - }; - if !enabled { - return Ok(false); - } - let has_password = password.is_some_and(|p| !p.is_empty()); - if !has_password && !bind_is_loopback(bind_host) { - return Err(format!( - "RECACHED_REPL_ENABLE is set and RECACHED_BIND is '{bind_host}', but \ - RECACHED_REPL_PASSWORD is unset — refusing to start. The replication port serves the \ - entire keyspace to whoever connects, so on any interface reachable from the network \ - it must be authenticated. Set RECACHED_REPL_PASSWORD, or bind to 127.0.0.1." - )); + if global_password.is_some() { + info!("Authentication ENABLED. Clients must send 'AUTH '."); + } else { + warn!("Authentication DISABLED. Set RECACHED_PASSWORD to enable."); } - Ok(true) -} -/// Parse `RECACHED_ALLOWED_ORIGINS` into exact origins. -/// -/// An origin is scheme + host + optional port and nothing else, so an entry -/// carrying a path is a misunderstanding of what will be compared against — and -/// one that would silently never match. Reject it at startup, in the same -/// spirit as `parse_allow_ips`. -fn parse_allowed_origins(raw: &str) -> Result, String> { - let mut origins = Vec::new(); - for entry in raw.split(',') { - let trimmed = entry.trim().trim_end_matches('/'); - if trimmed.is_empty() { - continue; - } - // Sandboxed iframes and `file://` documents send the literal `null`. - // An operator may legitimately need to admit them. - if trimmed.eq_ignore_ascii_case("null") { - origins.push("null".to_string()); - continue; - } - let Some((scheme, authority)) = trimmed.split_once("://") else { - return Err(format!( - "RECACHED_ALLOWED_ORIGINS: '{trimmed}' is not an origin — it needs a scheme, e.g. \ - https://app.example.com." - )); - }; - if scheme.is_empty() || authority.is_empty() { - return Err(format!( - "RECACHED_ALLOWED_ORIGINS: '{trimmed}' is not an origin — expected \ - scheme://host[:port]." - )); - } - if authority.contains('/') { - return Err(format!( - "RECACHED_ALLOWED_ORIGINS: '{trimmed}' contains a path. An origin is \ - scheme://host[:port] only, and a browser will never send a path — this entry \ - could never match." - )); - } - origins.push(trimmed.to_ascii_lowercase()); - } - if origins.is_empty() { - return Err( - "RECACHED_ALLOWED_ORIGINS is set but lists no origins — this would reject every \ - browser. Unset it to accept all origins." - .to_string(), + // ── sync scoping ────────────────────────────────────────────────────── + let sync_secret: Arc> = Arc::new( + std::env::var("RECACHED_SYNC_SECRET") + .ok() + .filter(|s| !s.is_empty()), + ); + if sync_secret.is_some() { + info!( + "Sync scoping ENABLED (strict): WebSocket clients receive no pushes and no key access until they present 'SYNC TOKEN '." + ); + } else { + warn!( + "Sync scoping DISABLED: every WebSocket client receives every mutation. Set RECACHED_SYNC_SECRET before exposing port {} to untrusted clients.", + ws_port ); } - Ok(origins) -} -/// Whether a WebSocket handshake carrying `origin` may proceed. -/// -/// Browsers apply neither CORS nor a preflight to WebSockets, so without this -/// check any page a user visits can open a socket to a reachable Recached and -/// act with that user's network position. On the common `ws://localhost:6380` -/// development setup that means every site in every tab. -/// -/// `Origin` is not a boundary against a native client, which simply omits the -/// header — and that is why an absent origin is allowed. What it does -/// distinguish is "the application I deployed" from "some other page in the same -/// browser", which is precisely the threat this port faces. An unset allowlist -/// permits everything, matching how an unset `RECACHED_PASSWORD` behaves. -fn origin_allowed(allowed: Option<&[String]>, origin: Option<&str>) -> bool { - let Some(list) = allowed else { - return true; - }; - let Some(origin) = origin else { - return true; + // ── IP allowlist ────────────────────────────────────────────────────── + let allowed_ips: Option>> = match std::env::var("RECACHED_ALLOW_IPS").ok() { + None => None, + Some(raw) => match parse_allow_ips(&raw) { + Ok(ips) => Some(Arc::new(ips)), + Err(msg) => { + error!("{msg}"); + std::process::exit(1); + } + }, }; - let origin = origin.trim().trim_end_matches('/'); - list.iter().any(|a| a.eq_ignore_ascii_case(origin)) -} -/// How long a connection may take to complete its TLS and/or WebSocket -/// handshake. Override: `RECACHED_HANDSHAKE_TIMEOUT` (seconds). -/// -/// The connection permit is taken before the handshake runs, so without a -/// deadline a client that opens a socket and then says nothing holds one of -/// `RECACHED_MAX_CONNECTIONS` slots indefinitely. A thousand such sockets cost -/// an attacker nothing and stop the server accepting real clients. -fn handshake_timeout() -> Duration { - static V: std::sync::OnceLock = std::sync::OnceLock::new(); - Duration::from_secs(*V.get_or_init(|| { - env_limit( - "RECACHED_HANDSHAKE_TIMEOUT", - DEFAULT_HANDSHAKE_TIMEOUT_SECS as usize, - ) as u64 - })) -} - -fn parse_memory_bytes(s: &str) -> Option { - let s = s.trim().to_lowercase(); - if let Some(n) = s.strip_suffix("gb") { - n.trim() - .parse::() - .ok() - .map(|n| n * 1024 * 1024 * 1024) - } else if let Some(n) = s.strip_suffix("mb") { - n.trim().parse::().ok().map(|n| n * 1024 * 1024) - } else if let Some(n) = s.strip_suffix("kb") { - n.trim().parse::().ok().map(|n| n * 1024) + if let Some(ips) = &allowed_ips { + info!("IP allowlist ENABLED: {:?}", ips); } else { - s.parse().ok() + warn!("IP allowlist DISABLED. Accepting all connections."); } -} -async fn save_snapshot(store: &KeyValueStore, cfg: &SnapshotConfig) { - let entries = store.snapshot(); - let count = entries.len(); - let tmp = temp_sibling(&cfg.path, "snap"); - match rmp_serde::to_vec(&entries) { - Err(e) => warn!("Snapshot serialize failed: {}", e), - Ok(bytes) => match write_private(&tmp, &bytes).await { - Err(e) => warn!("Snapshot write failed: {}", e), - Ok(()) => match tokio::fs::rename(&tmp, &cfg.path).await { - Err(e) => warn!("Snapshot rename failed: {}", e), - Ok(()) => { - sync_parent_dir(&cfg.path).await; - cfg.last_save.store(now_unix_secs(), Ordering::Relaxed); - info!("Snapshot saved: {} entries → {:?}", count, cfg.path); + // ── WebSocket origin allowlist ──────────────────────────────────────── + let allowed_origins: Arc>> = + Arc::new(match std::env::var("RECACHED_ALLOWED_ORIGINS").ok() { + None => None, + Some(raw) => match parse_allowed_origins(&raw) { + Ok(list) => Some(list), + Err(msg) => { + error!("{msg}"); + std::process::exit(1); } }, - }, - } -} + }); -async fn load_snapshot(store: &KeyValueStore, path: &std::path::Path) -> bool { - match tokio::fs::read(path).await { - Err(e) if e.kind() == std::io::ErrorKind::NotFound => { - info!("No snapshot at {:?}, starting fresh", path); - false - } - Err(e) => { - warn!("Snapshot read failed: {}", e); - false - } - Ok(bytes) => match rmp_serde::from_slice::>(&bytes) { - Err(e) => { - warn!("Snapshot deserialize failed: {}", e); - false - } - Ok(entries) => { - let count = entries.len(); - store.restore(entries); - info!("Snapshot loaded: {} entries ← {:?}", count, path); - true - } - }, + if let Some(list) = allowed_origins.as_ref() { + info!("WebSocket origin allowlist ENABLED: {:?}", list); + } else { + warn!( + "WebSocket origin allowlist DISABLED. Any web page a user visits can open a socket to \ + port {} — browsers apply neither CORS nor a preflight to WebSockets. Set \ + RECACHED_ALLOWED_ORIGINS before exposing this port to a browser.", + ws_port + ); } -} -// ── AOF ─────────────────────────────────────────────────────────────────────── - -#[derive(Clone, Copy, PartialEq)] -enum AofSync { - Always, - EverySec, - No, -} + // ── store ───────────────────────────────────────────────────────────── + let max_keys = std::env::var("RECACHED_MAX_KEYS") + .ok() + .and_then(|v| v.parse::().ok()); -struct AofWriter { - #[allow(dead_code)] - path: PathBuf, - file: tokio::sync::Mutex, - sync: AofSync, -} + let max_memory_bytes = std::env::var("RECACHED_MAX_MEMORY") + .ok() + .and_then(|v| parse_memory_bytes(&v)); -impl AofWriter { - async fn open(path: PathBuf, sync: AofSync) -> std::io::Result { - let mut opts = tokio::fs::OpenOptions::new(); - opts.create(true).append(true); - #[cfg(unix)] - opts.mode(0o600); - let file = opts.open(&path).await?; - // An AOF written by an earlier version is likely to be 0644 — tighten it - // on open, since `mode()` only applies to files this call creates. - #[cfg(unix)] - restrict_permissions(&file).await; - Ok(Self { - path, - file: tokio::sync::Mutex::new(file), - sync, - }) - } + let eviction_policy = match std::env::var("RECACHED_EVICTION") + .unwrap_or_default() + .to_lowercase() + .as_str() + { + "allkeys-lru" | "lru" => EvictionPolicy::AllKeysLru, + "allkeys-random" | "random" => EvictionPolicy::AllKeysRandom, + "volatile-lru" => EvictionPolicy::VolatileLru, + "volatile-ttl" | "ttl" => EvictionPolicy::VolatileTtl, + _ => EvictionPolicy::NoEviction, + }; - async fn append(&self, resp: &[u8]) { - let mut f = self.file.lock().await; - if f.write_all(resp).await.is_err() { - warn!("AOF write failed"); - return; - } - if self.sync == AofSync::Always { - // `flush()` alone only pushes tokio's buffer into a `write` syscall, - // which leaves the bytes in the page cache — surviving a process - // crash but not a power loss or kernel panic. `always` exists - // precisely to survive the latter, so it has to reach the device. - if let Err(e) = f.flush().await { - warn!("AOF flush failed: {}", e); - return; - } - if let Err(e) = f.sync_data().await { - warn!("AOF fsync failed: {}", e); - } - } + if max_keys.is_some() || max_memory_bytes.is_some() { + info!( + "Key limit: {:?}, memory limit: {:?} bytes, eviction: {:?}", + max_keys, max_memory_bytes, eviction_policy + ); } - /// Flush and fsync. Called on the `everysec` ticker and before shutdown. - /// - /// `sync_data` rather than `sync_all`: the AOF is append-only, so its - /// metadata beyond the length carries nothing worth an extra barrier. - async fn flush(&self) { - let mut f = self.file.lock().await; - if let Err(e) = f.flush().await { - warn!("AOF flush failed: {}", e); - return; - } - if let Err(e) = f.sync_data().await { - warn!("AOF fsync failed: {}", e); - } - } + let mut store_inner = KeyValueStore::with_config(max_keys, max_memory_bytes, eviction_policy); + // Eviction sample size — the knob Redis exposes as `maxmemory-samples`. + // Configured before the store is shared, so no interior mutability is needed. + store_inner.set_eviction_sample(env_limit("RECACHED_EVICTION_SAMPLE", 10)); + let store = Arc::new(store_inner); - async fn truncate(&self) { - let f = self.file.lock().await; - match f.set_len(0).await { - // The truncation itself must be durable, or a crash can resurrect a - // log the snapshot has already subsumed and replay it on top. - Ok(()) => match f.sync_all().await { - Ok(()) => info!("AOF truncated after snapshot save"), - Err(e) => warn!("AOF truncate fsync failed: {}", e), - }, - Err(e) => warn!("AOF truncate failed: {}", e), - } - } -} + // ── snapshot persistence ────────────────────────────────────────────── + let save_path = PathBuf::from( + std::env::var("RECACHED_SAVE_PATH").unwrap_or_else(|_| "recached.rdb".to_string()), + ); -async fn replay_aof(store: &KeyValueStore, path: &std::path::Path) -> usize { - let bytes = match tokio::fs::read(path).await { - Err(e) if e.kind() == ErrorKind::NotFound => return 0, - Err(e) => { - warn!("AOF read failed: {}", e); - return 0; + // RECACHED_SAVE takes priority: "900:1,300:10,60:10000" (secs:changes pairs). + // Falls back to RECACHED_SAVE_INTERVAL (single-condition, 1 change required). + let save_conditions: Vec = if let Ok(s) = std::env::var("RECACHED_SAVE") { + parse_save_conditions(&s) + } else { + let interval: u64 = std::env::var("RECACHED_SAVE_INTERVAL") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(900); + if interval > 0 { + vec![SaveCondition { + secs: interval, + changes: 1, + }] + } else { + vec![] } - Ok(b) => b, }; - let mut replayed = 0usize; - let mut offset = 0; - while offset < bytes.len() { - match Value::parse(&bytes[offset..]) { - Ok((value, consumed)) => { - offset += consumed; - // Writes are recorded via `on_write` in RESP3 Push form (`>N`); - // normalise to Array so Command::from_value can parse them. - let normalised = match value { - Value::Push(inner) => Value::Array(Some(inner)), - other => other, - }; - if let Ok(cmd) = Command::from_value(normalised) { - store.execute(cmd); - replayed += 1; + + load_snapshot(&store, &save_path).await; + + let snap_cfg = Arc::new(SnapshotConfig { + path: save_path, + last_save: AtomicI64::new(now_unix_secs()), + }); + + // ── AOF ─────────────────────────────────────────────────────────────── + let aof_path = std::env::var("RECACHED_AOF_PATH").ok().map(PathBuf::from); + let aof_sync = match std::env::var("RECACHED_AOF_SYNC") + .unwrap_or_default() + .to_lowercase() + .as_str() + { + "always" => AofSync::Always, + "no" => AofSync::No, + _ => AofSync::EverySec, + }; + + let aof: Option> = if let Some(path) = aof_path { + match AofWriter::open(path.clone(), aof_sync).await { + Ok(w) => { + replay_aof(&store, &path).await; + let writer = Arc::new(w); + if aof_sync == AofSync::EverySec { + let w2 = Arc::clone(&writer); + tokio::spawn(async move { + let mut interval = + tokio::time::interval(tokio::time::Duration::from_secs(1)); + loop { + interval.tick().await; + w2.flush().await; + } + }); + } + info!( + "AOF enabled: {:?} (sync={})", + path, + match aof_sync { + AofSync::Always => "always", + AofSync::EverySec => "everysec", + AofSync::No => "no", + } + ); + // `always` fsyncs inside the AOF lock, so every write in the + // process waits for one disk barrier — measured at roughly + // 20 ms per append on APFS, i.e. tens of writes per second + // rather than tens of thousands. That is the honest cost of the + // guarantee, but an operator who picked it casually will read + // the result as a hang, so say so at startup. + if aof_sync == AofSync::Always { + warn!( + "RECACHED_AOF_SYNC=always fsyncs on every write and serialises all writers \ + behind it — expect write throughput in the tens per second. Use everysec \ + unless you genuinely cannot lose one second of writes." + ); } + Some(writer) } - Err(e) if e.is_incomplete() => break, - Err(_) => { - warn!("AOF corrupted at offset {}, stopping replay", offset); - break; + Err(e) => { + warn!("AOF open failed: {} — running without AOF", e); + None } } - } - if replayed > 0 { - info!("AOF replayed: {} commands ← {:?}", replayed, path); - } - replayed -} - -// ── Replication ─────────────────────────────────────────────────────────────── + } else { + None + }; -type ReplSender = mpsc::Sender>; + // ── TLS ─────────────────────────────────────────────────────────────── + // Resolved before replication: the replication listener uses the same + // certificate, so it has to exist before that listener is spawned. + let tls_acceptor: Option = load_tls_acceptor(); + if tls_acceptor.is_some() { + info!( + "TLS ENABLED (cert={}, key={})", + std::env::var("RECACHED_TLS_CERT").unwrap_or_default(), + std::env::var("RECACHED_TLS_KEY").unwrap_or_default() + ); + } else { + warn!("TLS DISABLED. Set RECACHED_TLS_CERT and RECACHED_TLS_KEY to enable."); + } + let tls_acceptor = Arc::new(tls_acceptor); -/// A connected replica: its write channel plus the counters that make lag -/// observable. -/// -/// Replication was previously one-way, so the primary could only report how -/// many replicas were attached — never how far behind one had fallen. The -/// replica now acknowledges each applied frame, and the difference between -/// what was queued and what was acknowledged is the lag. -struct ReplicaHandle { - tx: ReplSender, - /// Frames handed to this replica's channel. - sent: Arc, - /// Frames the replica reports as applied. - acked: Arc, -} + // ── connection limiter ──────────────────────────────────────────────── + // Resolved before replication because the replication listener shares this + // budget: a flood of replica connections must not be able to starve real + // clients, and `maxclients` should mean the total. + let max_connections = std::env::var("RECACHED_MAX_CONNECTIONS") + .ok() + .and_then(|v| v.parse::().ok()) + .unwrap_or(DEFAULT_MAX_CONNECTIONS); + info!("Max connections: {}", max_connections); + let semaphore = Arc::new(Semaphore::new(max_connections)); -/// Connected-replica registry. `count` mirrors `senders.len()` (updated by -/// every writer while holding the lock) so the per-write hot path can skip -/// the mutex entirely when no replica is connected. -struct ReplHub { - senders: tokio::sync::Mutex>, - count: AtomicUsize, -} + // ── Replication ─────────────────────────────────────────────────────── + let replicaof = std::env::var("RECACHED_REPLICAOF").ok(); + let repl_port: u16 = std::env::var("RECACHED_REPL_PORT") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(6381); + let repl_password: Option = std::env::var("RECACHED_REPL_PASSWORD").ok(); + let repl_channel_capacity: usize = std::env::var("RECACHED_REPL_BUFFER") + .ok() + .and_then(|v| v.parse().ok()) + .filter(|&n: &usize| n > 0) + .unwrap_or(DEFAULT_REPL_CHANNEL_CAPACITY); + let failover_timeout_secs: Option = std::env::var("RECACHED_FAILOVER_TIMEOUT") + .ok() + .and_then(|v| v.parse().ok()) + .filter(|&n| n > 0); -impl ReplHub { - /// Deepest send queue across connected replicas, in frames. - /// - /// Replication is fire-and-forget — replicas never acknowledge an applied - /// offset — so true offset lag is not observable without a protocol change. - /// Queue depth is the honest proxy available today: a replica that cannot - /// keep up backs its channel up, and a queue at capacity means frames are - /// about to be dropped. - async fn max_queue_depth(&self) -> usize { - let senders = self.senders.lock().await; - senders - .iter() - .map(|r| r.tx.max_capacity().saturating_sub(r.tx.capacity())) - .max() - .unwrap_or(0) - } + // Whether to bind the replication listener at all. Refuses to start on a + // network-reachable interface without a password rather than serving the + // keyspace unauthenticated. + let repl_listen = match resolve_repl_listen( + std::env::var("RECACHED_REPL_ENABLE").ok(), + &bind_host, + repl_password.as_deref(), + ) { + Ok(v) => v, + Err(msg) => { + error!("{msg}"); + std::process::exit(1); + } + }; - /// Send one frame to every attached replica, dropping any that cannot keep - /// up. Increments each surviving replica's sent counter, which is one half - /// of the lag calculation. - async fn fan_out(&self, bytes: Vec) { - let mut reg = self.senders.lock().await; - reg.retain(|r| match r.tx.try_send(bytes.clone()) { - Ok(()) => { - r.sent.fetch_add(1, Ordering::Relaxed); - true - } - Err(mpsc::error::TrySendError::Full(_)) => { - warn!( - "Replica fell too far behind (channel full) — disconnecting so it can resync" + // Outbound replication TLS. Configured separately from the listener's TLS + // because the two directions are independent: this node may serve replicas + // over TLS, follow a primary over TLS, both, or neither. + let repl_tls: Option<(TlsConnector, String)> = match std::env::var("RECACHED_REPL_TLS_CA") + .ok() + .filter(|s| !s.is_empty()) + { + None => None, + Some(ca) => match load_repl_tls_connector(&ca) { + Ok(connector) => { + let servername = repl_tls_servername( + replicaof.as_deref().unwrap_or_default(), + std::env::var("RECACHED_REPL_TLS_SERVERNAME").ok(), ); - false + info!( + "Replication client TLS ENABLED (CA={}, verifying primary as '{}')", + ca, servername + ); + Some((connector, servername)) } - Err(mpsc::error::TrySendError::Closed(_)) => false, - }); - self.count.store(reg.len(), Ordering::Relaxed); - } - - /// Frames the furthest-behind replica has yet to acknowledge. - /// - /// This is true lag: how much of what the primary sent has actually been - /// applied downstream. Queue depth only shows what is stuck locally, and - /// reads zero for a replica that has received frames but cannot apply them. - async fn max_lag_frames(&self) -> u64 { - let senders = self.senders.lock().await; - senders - .iter() - .map(|r| { - r.sent - .load(Ordering::Relaxed) - .saturating_sub(r.acked.load(Ordering::Relaxed)) - }) - .max() - .unwrap_or(0) - } + Err(msg) => { + error!("{msg}"); + std::process::exit(1); + } + }, + }; - fn new() -> ReplRegistry { - Arc::new(ReplHub { - senders: tokio::sync::Mutex::new(Vec::new()), - count: AtomicUsize::new(0), - }) + if replicaof.is_some() && repl_tls.is_none() { + warn!( + "Replication to the primary is PLAINTEXT — the password and the entire keyspace cross \ + the network unencrypted, and the primary's identity is not verified. Set \ + RECACHED_REPL_TLS_CA, or keep replication on a private network." + ); } - fn is_empty(&self) -> bool { - self.count.load(Ordering::Relaxed) == 0 + if !repl_listen { + info!( + "Replication server DISABLED — port {} is not bound. Set RECACHED_REPL_ENABLE=1 on any \ + node that serves replicas (including a replica serving sub-replicas).", + repl_port + ); + } else if repl_password.is_some() { + info!( + "Replication server ENABLED on port {} with auth (RECACHED_REPL_PASSWORD is set).", + repl_port + ); + } else { + warn!( + "Replication server ENABLED on port {} WITHOUT a password, on loopback only. It serves \ + the entire keyspace to whoever connects — set RECACHED_REPL_PASSWORD before binding \ + any other interface.", + repl_port + ); } -} -type ReplRegistry = Arc; + let is_replica_start = replicaof.is_some(); + let replicas: ReplRegistry = ReplHub::new(); -/// Default per-replica channel capacity (number of pending write frames). -/// When a replica falls this many writes behind the primary it is disconnected -/// so it can reconnect and receive a fresh snapshot — the primary write path -/// is never blocked. -const DEFAULT_REPL_CHANNEL_CAPACITY: usize = 4096; + // ── server state ────────────────────────────────────────────────────── + let state = Arc::new(ServerState { + snap: Arc::clone(&snap_cfg), + aof, + replicas: Arc::clone(&replicas), + is_replica: std::sync::atomic::AtomicBool::new(is_replica_start), + dedup: std::sync::Mutex::new(HashMap::new()), + ephemeral: std::sync::Mutex::new(HashMap::new()), + dedup_dirty: std::sync::atomic::AtomicBool::new(false), + }); -/// Upper bound on a single length-prefixed replication frame (snapshot or -/// command). The replication port may be unauthenticated and plaintext, so an -/// untrusted peer could otherwise send a 4 GB length prefix and force a matching -/// allocation. 512 MB comfortably covers a large snapshot while bounding abuse. -const MAX_REPL_FRAME_BYTES: usize = 512 * 1024 * 1024; + // Restore exactly-once bookkeeping before accepting connections, so a + // client replaying an unacknowledged write after a restart is recognised + // rather than applied twice. + state.load_dedup().await; -// ── Server state ────────────────────────────────────────────────────────────── - -struct ServerState { - snap: Arc, - aof: Option>, - replicas: ReplRegistry, - /// true = currently acting as a read-only replica - is_replica: std::sync::atomic::AtomicBool, - /// Exactly-once bookkeeping for DEDUP-wrapped writes: client id → - /// (highest id applied, last-seen ms). Clients send monotonically - /// increasing ids and replay in order, so a single high-water mark per - /// client suffices — no seen-set. In-memory only: a server restart - /// reopens the (already narrow) duplicate window, which is documented. - dedup: std::sync::Mutex>, - /// Ephemeral (`ESET`) keys → the connection that currently owns them. - /// - /// Ownership transfers on each `ESET`, which is what makes multiple tabs - /// work: two tabs both setting `presence:user:42` leave the *later* one as - /// owner, so the first tab closing does not mark the user offline. Only the - /// owning connection's close deletes the key. - ephemeral: std::sync::Mutex>, - /// Set when a dedup high-water mark advances; cleared once persisted. - dedup_dirty: std::sync::atomic::AtomicBool, -} - -impl ServerState { - /// Record `conn_id` as the owner of an ephemeral key, replacing any - /// previous owner. - fn claim_ephemeral(&self, key: &str, conn_id: u64) { - if let Ok(mut map) = self.ephemeral.lock() { - map.insert(key.to_string(), conn_id); - } - } - - /// Keys still owned by `conn_id`, removed from the registry. Called once - /// when a connection closes. - fn take_ephemeral_for(&self, conn_id: u64) -> Vec { - let Ok(mut map) = self.ephemeral.lock() else { - return Vec::new(); - }; - let owned: Vec = map - .iter() - .filter(|(_, id)| **id == conn_id) - .map(|(k, _)| k.clone()) - .collect(); - for k in &owned { - map.remove(k); - } - owned - } -} - -/// Sweep dedup client entries idle longer than this once the map is large. -const DEDUP_IDLE_MS: u64 = 24 * 60 * 60 * 1000; -const DEDUP_SWEEP_THRESHOLD: usize = 10_000; - -impl ServerState { - fn is_replica(&self) -> bool { - self.is_replica.load(Ordering::Relaxed) - } - - fn promote_to_primary(&self) { - self.is_replica.store(false, Ordering::Relaxed); - info!("REPLICAOF NO ONE: promoted to primary — writes now accepted"); - } - - /// True when a write must be RESP-encoded for the durability/replication - /// path even if no other consumer needs it. - fn needs_write_log(&self) -> bool { - self.aof.is_some() || !self.replicas.is_empty() + // ── Dedup flush ─────────────────────────────────────────────────────── + // The map is one u64 per client, so it can be persisted far more often than + // the snapshot. This bounds the duplicate window on an unclean shutdown to + // roughly this interval rather than to the snapshot cadence. + { + let state_dedup = Arc::clone(&state); + tokio::spawn(async move { + let mut ticker = tokio::time::interval(tokio::time::Duration::from_secs(1)); + ticker.tick().await; + loop { + ticker.tick().await; + state_dedup.persist_dedup().await; + } + }); } - /// Record a DEDUP-wrapped write. Returns `true` when `id` was already - /// applied for this client (the write must be skipped). Marks the id - /// *before* execution so a crash between check and execute can never - /// double-apply. - fn dedup_seen(&self, client: &str, id: u64) -> bool { - let now = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_millis() as u64; - let mut map = self.dedup.lock().expect("dedup mutex poisoned"); - if map.len() > DEDUP_SWEEP_THRESHOLD { - map.retain(|_, (_, seen)| now.saturating_sub(*seen) < DEDUP_IDLE_MS); - } - match map.get_mut(client) { - Some((hwm, seen)) => { - *seen = now; - if id <= *hwm { - true - } else { - *hwm = id; - self.dedup_dirty.store(true, Ordering::Relaxed); - false + // ── autosave ────────────────────────────────────────────────────────── + if !save_conditions.is_empty() { + let store_snap = Arc::clone(&store); + let state_snap = Arc::clone(&state); + let conditions = save_conditions; + tokio::spawn(async move { + let mut ticker = tokio::time::interval(tokio::time::Duration::from_secs(1)); + ticker.tick().await; // skip immediate first tick + loop { + ticker.tick().await; + let now = now_unix_secs(); + let last = state_snap.snap.last_save.load(Ordering::Relaxed); + let elapsed = now.saturating_sub(last).max(0) as u64; + let dirty = store_snap.dirty_count(); + if dirty > 0 + && conditions + .iter() + .any(|c| elapsed >= c.secs && dirty >= c.changes) + { + state_snap.save(&store_snap).await; } } - None => { - map.insert(client.to_string(), (id, now)); - self.dedup_dirty.store(true, Ordering::Relaxed); - false - } - } + }); + info!("Autosave active → {:?}", snap_cfg.path); + } else { + info!( + "Autosave disabled (RECACHED_SAVE=0 or RECACHED_SAVE_INTERVAL=0). Use SAVE or BGSAVE manually." + ); } - /// Called after every successful write: appends to AOF and fans out to replicas. - async fn on_write(&self, resp: &[u8]) { - if let Some(aof) = &self.aof { - aof.append(resp).await; - } - if self.replicas.is_empty() { - return; - } - self.replicas.fan_out(resp.to_vec()).await; - } + // ── broadcast channel (mutation sync) ──────────────────────────────── + // Carries (sender_conn_id, resp_encoded_mutation). WS receivers skip their + // own messages. Created before replication so a replica can push the writes + // it receives from the primary to its own local WebSocket clients. + let (tx, _rx) = broadcast::channel::(BROADCAST_CHANNEL_CAPACITY); - /// Path of the dedup sidecar, alongside the snapshot. - fn dedup_path(&self) -> std::path::PathBuf { - self.snap.path.with_extension("dedup") + // ── start replication ───────────────────────────────────────────────── + // Opt-in. The listener may run on a replica as well as a primary, so a + // replica can serve sub-replicas (multi-tier replication) — but it is a + // decision the operator makes, not a port that appears by default. + if repl_listen { + let store_r = Arc::clone(&store); + let snap_r = Arc::clone(&snap_cfg); + let reg_r = Arc::clone(&replicas); + let pwd_r = repl_password.clone().map(Arc::new); + let cap_r = repl_channel_capacity; + let host_r = bind_host.clone(); + let allowed_r = allowed_ips.clone(); + let sem_r = Arc::clone(&semaphore); + let thr_r = ReplAuthThrottle::new(); + let tls_r = Arc::clone(&tls_acceptor); + tokio::spawn(async move { + run_repl_server( + host_r, repl_port, store_r, snap_r, reg_r, pwd_r, cap_r, allowed_r, sem_r, thr_r, + tls_r, + ) + .await; + }); } - - /// Persist dedup high-water marks so exactly-once delivery survives a - /// restart. Written atomically (temp + rename) and only when a mark has - /// advanced. The map is one `u64` per client, so this stays small enough to - /// flush far more often than the snapshot. - async fn persist_dedup(&self) { - if !self.dedup_dirty.swap(false, Ordering::Relaxed) { - return; - } - let marks: Vec<(String, u64)> = match self.dedup.lock() { - Ok(map) => map.iter().map(|(c, (hwm, _))| (c.clone(), *hwm)).collect(), - Err(_) => return, - }; - let path = self.dedup_path(); - let tmp = temp_sibling(&path, "dedup"); - match rmp_serde::to_vec(&marks) { - Err(e) => warn!("Dedup serialize failed: {}", e), - Ok(bytes) => match write_private(&tmp, &bytes).await { - Err(e) => warn!("Dedup write failed: {}", e), - Ok(()) => match tokio::fs::rename(&tmp, &path).await { - Err(e) => warn!("Dedup rename failed: {}", e), - Ok(()) => sync_parent_dir(&path).await, - }, - }, + if is_replica_start && let Some(primary_addr) = replicaof { + let store_r = Arc::clone(&store); + let state_r = Arc::clone(&state); + let pwd_r = repl_password.clone(); + let fo_r = failover_timeout_secs; + let tx_r = tx.clone(); + let tls_r = repl_tls; + tokio::spawn(async move { + run_repl_client(primary_addr, store_r, state_r, pwd_r, fo_r, tx_r, tls_r).await; + }); + if let Some(t) = failover_timeout_secs { + info!( + "Running as replica — auto-failover enabled (promotes after {}s of primary being unreachable)", + t + ); + } else { + info!( + "Running as replica — write commands will be rejected (auto-failover disabled; set RECACHED_FAILOVER_TIMEOUT to enable)" + ); } } - /// Restore dedup marks at boot. `seen` timestamps are not persisted — they - /// only drive idle sweeping, so restored entries start their idle clock now. - async fn load_dedup(&self) { - let path = self.dedup_path(); - let Ok(bytes) = tokio::fs::read(&path).await else { - return; - }; - match rmp_serde::from_slice::>(&bytes) { - Err(e) => warn!("Dedup sidecar unreadable ({}), ignoring: {:?}", e, path), - Ok(marks) => { - let now = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_millis() as u64; - if let Ok(mut map) = self.dedup.lock() { - let count = marks.len(); - for (client, hwm) in marks { - map.insert(client, (hwm, now)); - } - info!("Restored {} dedup high-water mark(s)", count); - } + // ── background eviction ─────────────────────────────────────────────── + { + let store_sweep = Arc::clone(&store); + tokio::spawn(async move { + let mut interval = + tokio::time::interval(tokio::time::Duration::from_secs(EVICTION_INTERVAL_SECS)); + loop { + interval.tick().await; + store_sweep.sweep_expired(); + store_sweep.try_evict_for_memory(); } - } - } - - /// Save snapshot, reset the dirty counter, then truncate AOF (snapshot subsumes the log). - async fn save(&self, store: &KeyValueStore) { - self.persist_dedup().await; - save_snapshot(store, &self.snap).await; - store.reset_dirty(); - if let Some(aof) = &self.aof { - aof.truncate().await; - } - } -} - -fn is_write_command(cmd: &Command) -> bool { - if let Command::Dedup(_, _, inner) = cmd { - return is_write_command(inner); + }); } - matches!( - cmd, - Command::Set(..) - | Command::ESet(..) - | Command::Del(..) - | Command::Unlink(..) - | Command::Append(..) - | Command::GetSet(..) - | Command::MSet(..) - | Command::SetNx(..) - | Command::SetEx(..) - | Command::PSetEx(..) - | Command::Incr(..) - | Command::Decr(..) - | Command::IncrBy(..) - | Command::DecrBy(..) - | Command::Expire(..) - | Command::PExpire(..) - | Command::ExpireAt(..) - | Command::PExpireAt(..) - | Command::Persist(..) - | Command::FlushDb - | Command::Rename(..) - | Command::HSet(..) - | Command::HDel(..) - | Command::HIncrBy(..) - | Command::HIncrByFloat(..) - | Command::HSetNx(..) - | Command::LPush(..) - | Command::RPush(..) - | Command::LPushX(..) - | Command::RPushX(..) - | Command::LPop(..) - | Command::RPop(..) - | Command::LSet(..) - | Command::LRem(..) - | Command::LTrim(..) - | Command::SAdd(..) - | Command::SRem(..) - | Command::SInterStore(..) - | Command::SUnionStore(..) - | Command::SDiffStore(..) - | Command::SPop(..) - | Command::SMove(..) - | Command::ZAdd(..) - | Command::ZRem(..) - | Command::ZIncrBy(..) - | Command::RlSet(..) - | Command::RlCheck(..) - | Command::JSet(..) - | Command::JMerge(..) - ) -} - -// ── Save conditions ─────────────────────────────────────────────────────────── - -/// A single autosave condition: save if `changes` or more writes have -/// accumulated within `secs` seconds of the last save. -struct SaveCondition { - secs: u64, - changes: u64, -} - -/// Parse `RECACHED_SAVE` value: comma-separated `seconds:changes` pairs. -/// Example: `"900:1,300:10,60:10000"` → save after 1 change in 15 min, -/// 10 changes in 5 min, or 10 000 changes in 1 min — whichever comes first. -fn parse_save_conditions(s: &str) -> Vec { - s.split(',') - .filter_map(|pair| { - let mut parts = pair.trim().splitn(2, ':'); - let secs: u64 = parts.next()?.trim().parse().ok()?; - let changes: u64 = parts.next()?.trim().parse().ok()?; - Some(SaveCondition { secs, changes }) - }) - .collect() -} -// ── Replication server (primary side) ──────────────────────────────────────── + // ── pub/sub hub ─────────────────────────────────────────────────────── + let pubsub: SharedPubSub = Arc::new(tokio::sync::Mutex::new(PubSubHub::new())); -/// Per-peer replication auth throttle. -/// -/// The RESP port drops a connection after `MAX_AUTH_FAILURES` guesses, but the -/// replication handshake is one-shot: a wrong password costs the attacker a -/// single TCP connection and nothing else, so the port offered effectively -/// unlimited guesses at a secret that yields the entire keyspace. Failures are -/// counted per source address over a rolling window, and a peer that exhausts -/// them is refused before the handshake is read at all. -/// -/// Keyed by address rather than by connection, which is the whole point — the -/// weakness being closed is that reconnecting reset the count. -struct ReplAuthThrottle { - failures: std::sync::Mutex>, -} + // ── watch registry ──────────────────────────────────────────────────── + let watch_registry: WatchRegistry = WatchHub::new(); -impl ReplAuthThrottle { - fn new() -> Arc { - Arc::new(Self { - failures: std::sync::Mutex::new(HashMap::new()), - }) + // ── Capacity & sync metrics ─────────────────────────────────────────── + // Traffic counters are event-driven, but capacity is a level, not an event: + // memory, key count and eviction rate have to be sampled. Without these an + // operator cannot answer "am I near the cap?" or "is eviction thrashing?" + // from a dashboard — see docs/server/operations.md. + { + let store_m = Arc::clone(&store); + let state_m = Arc::clone(&state); + let registry_m = watch_registry.clone(); + tokio::spawn(async move { + let mut ticker = tokio::time::interval(tokio::time::Duration::from_secs(5)); + loop { + ticker.tick().await; + // One walk of the keyspace feeds both gauges and the cached + // sample INFO reads, instead of one walk per number. + let sample = store_m.keyspace_sample(); + store_sampled_keyspace(sample); + gauge!("recached_memory_bytes").set(sample.memory_bytes as f64); + gauge!("recached_keys").set(sample.keys as f64); + counter!("recached_evictions_total").absolute(store_m.evicted_count()); + gauge!("recached_replicas_connected") + .set(state_m.replicas.count.load(Ordering::Relaxed) as f64); + gauge!("recached_replication_queue_depth") + .set(state_m.replicas.max_queue_depth().await as f64); + gauge!("recached_replication_lag_frames") + .set(state_m.replicas.max_lag_frames().await as f64); + gauge!("recached_live_queries") + .set(registry_m.watched_patterns.load(Ordering::Relaxed) as f64); + gauge!("recached_watched_keys") + .set(registry_m.watched_keys.load(Ordering::Relaxed) as f64); + gauge!("recached_dedup_clients_tracked") + .set(state_m.dedup.lock().map(|m| m.len()).unwrap_or(0) as f64); + } + }); } - /// True when this peer has spent its attempts and must be refused. - fn is_blocked(&self, ip: IpAddr) -> bool { - let Ok(map) = self.failures.lock() else { - return false; - }; - match map.get(&ip) { - Some((count, last)) => *count >= MAX_AUTH_FAILURES && last.elapsed() < REPL_AUTH_WINDOW, - None => false, - } - } + // Startup facts for INFO. Recorded once the configuration is fully + // resolved and before any listener binds, so no connection can observe the + // defaults. + let _ = SERVER_FACTS.set(ServerFacts { + start: SystemTime::now(), + run_id: generate_run_id(), + tcp_port, + ws_port, + max_connections, + tls_enabled: tls_acceptor.is_some(), + auth_enabled: global_password.is_some(), + aof_enabled: state.aof.is_some(), + }); - fn record_failure(&self, ip: IpAddr) { - let Ok(mut map) = self.failures.lock() else { - return; - }; - let now = std::time::Instant::now(); - // Sweep before inserting so a spray across many source addresses cannot - // grow the map without bound. - if map.len() >= REPL_AUTH_SWEEP_THRESHOLD { - map.retain(|_, (_, last)| last.elapsed() < REPL_AUTH_WINDOW); - } - let entry = map.entry(ip).or_insert((0, now)); - // A peer that went quiet for longer than the window starts over, so a - // slow trickle is not punished forever. - if entry.1.elapsed() >= REPL_AUTH_WINDOW { - *entry = (0, now); - } - entry.0 = entry.0.saturating_add(1); - entry.1 = now; - } + // ── listeners ───────────────────────────────────────────────────────── + let n_accept = num_cpus::get(); + let tcp_listeners = make_tcp_listeners(&format!("{}:{}", bind_host, tcp_port), n_accept)?; + info!( + "TCP server listening on {}:{} ({} accept loop(s))", + bind_host, tcp_port, n_accept + ); - fn record_success(&self, ip: IpAddr) { - if let Ok(mut map) = self.failures.lock() { - map.remove(&ip); - } - } -} + let ws_listener = TcpListener::bind(format!("{}:{}", bind_host, ws_port)).await?; + info!("WebSocket server listening on {}:{}", bind_host, ws_port); -/// Read the newline-terminated replication auth line. -/// -/// Reads until the terminator rather than reading exactly `password.len() + 1` -/// bytes, which is how the previous implementation worked: the number of bytes -/// the server waited for *was* the password length, so an attacker could -/// recover it exactly by drip-feeding one byte at a time and watching when the -/// server replied. -async fn read_repl_auth_line(socket: &mut S) -> std::io::Result> -where - S: AsyncRead + Unpin, -{ - let mut line = Vec::with_capacity(64); - let mut byte = [0u8; 1]; - loop { - socket.read_exact(&mut byte).await?; - if byte[0] == b'\n' { - return Ok(line); - } - if line.len() >= MAX_REPL_AUTH_LINE { - return Err(std::io::Error::new( - ErrorKind::InvalidData, - "replication auth line too long", - )); - } - line.push(byte[0]); - } -} + // Spawn one accept loop per CPU core, each with its own SO_REUSEPORT socket. + // The OS load-balances incoming connections across all loops. + for tcp_listener in tcp_listeners { + let store_tcp = Arc::clone(&store); + let tx_tcp = tx.clone(); + let pass_tcp = Arc::clone(&global_password); + let allowed_tcp = allowed_ips.clone(); + let sem_tcp = Arc::clone(&semaphore); + let pubsub_tcp = Arc::clone(&pubsub); + let tls_tcp = Arc::clone(&tls_acceptor); + let watch_tcp = Arc::clone(&watch_registry); + let snap_tcp = Arc::clone(&state); -#[allow(clippy::too_many_arguments)] -async fn run_repl_server( - bind_host: String, - port: u16, - store: Arc, - snap_cfg: Arc, - replicas: ReplRegistry, - repl_password: Option>, - repl_channel_capacity: usize, - allowed_ips: Option>>, - semaphore: Arc, - throttle: Arc, - tls: Arc>, -) { - let listener = match TcpListener::bind(format!("{}:{}", bind_host, port)).await { - Ok(l) => l, - Err(e) => { - warn!("Replication listener failed to bind :{}: {}", port, e); - return; - } - }; - info!( - "Replication server listening on {}:{} ({})", - bind_host, - port, - if tls.is_some() { "TLS" } else { "plaintext" } - ); - loop { - match listener.accept().await { - Ok((socket, addr)) => { - // The IP allowlist and the connection limit were previously - // applied on the RESP and WebSocket listeners only, so neither - // constrained the one port that streams the whole keyspace. - if let Some(allowed) = &allowed_ips - && !allowed.contains(&addr.ip()) - { - debug!("Replication: rejected IP {}", addr.ip()); - continue; - } - if throttle.is_blocked(addr.ip()) { - warn!( - "Replication: {} refused — too many failed auth attempts", - addr.ip() - ); - continue; - } - let permit = match Arc::clone(&semaphore).try_acquire_owned() { - Ok(p) => p, - Err(_) => { - warn!("Replication: connection limit reached, dropping {}", addr); - continue; - } - }; - info!("Replica connected from {}", addr); - let store = Arc::clone(&store); - let snap_cfg = Arc::clone(&snap_cfg); - let replicas = Arc::clone(&replicas); - let pwd = repl_password.clone(); - let thr = Arc::clone(&throttle); - let tls = Arc::clone(&tls); - tokio::spawn(async move { - let _permit = permit; - // Bounded like the other listeners: the permit is already - // held, so a peer that never negotiates must not keep it. - let outcome = if let Some(acceptor) = tls.as_ref() { - match tokio::time::timeout(handshake_timeout(), acceptor.accept(socket)) - .await + tokio::spawn(async move { + loop { + match tcp_listener.accept().await { + Ok((socket, addr)) => { + let _ = socket.set_nodelay(true); + if let Some(allowed) = &allowed_tcp + && !allowed.contains(&addr.ip()) { - Ok(Ok(stream)) => { - handle_replica( - stream, - store, - snap_cfg, - replicas, - pwd, - repl_channel_capacity, - addr.ip(), - thr, - ) - .await - } - Ok(Err(e)) => { - // The most likely cause by far is a replica that - // has not been given RECACHED_REPL_TLS_CA, which - // otherwise looks like an unexplained disconnect. - warn!( - "Replication TLS handshake failed from {}: {} — is that \ - replica configured with RECACHED_REPL_TLS_CA?", - addr, e - ); - return; - } + debug!("TCP: rejected IP {}", addr.ip()); + continue; + } + let permit = match Arc::clone(&sem_tcp).try_acquire_owned() { + Ok(p) => p, Err(_) => { - debug!("Replication TLS handshake from {} timed out", addr); - return; + warn!("TCP: connection limit reached, dropping {}", addr); + continue; } - } - } else { - handle_replica( - socket, - store, - snap_cfg, - replicas, - pwd, - repl_channel_capacity, - addr.ip(), - thr, - ) - .await - }; - if let Err(e) = outcome { - info!("Replica {} disconnected: {}", addr, e); - } - }); - } - Err(e) => warn!("Replication accept error: {}", e), - } - } -} - -#[allow(clippy::too_many_arguments)] -async fn handle_replica( - mut socket: S, - store: Arc, - _snap_cfg: Arc, - replicas: ReplRegistry, - repl_password: Option>, - repl_channel_capacity: usize, - peer_ip: IpAddr, - throttle: Arc, -) -> std::io::Result<()> -where - S: AsyncRead + AsyncWrite + Unpin + Send, -{ - // 0. Auth handshake — replica must send "\n" before anything else. - // - // Bounded by a deadline as well as a length: a peer that connects and then - // says nothing would otherwise hold its connection permit forever. - if let Some(pwd) = &repl_password { - let line = match tokio::time::timeout(handshake_timeout(), read_repl_auth_line(&mut socket)) - .await - { - Ok(res) => res?, - Err(_) => { - return Err(std::io::Error::new( - ErrorKind::TimedOut, - "replication auth handshake timed out", - )); - } - }; - if !ct_eq_bytes(&line, pwd.as_bytes()) { - throttle.record_failure(peer_ip); - let _ = socket - .write_all(b"-ERR invalid replication password\n") - .await; - return Err(std::io::Error::new( - ErrorKind::PermissionDenied, - "replication auth failed", - )); - } - throttle.record_success(peer_ip); - socket.write_all(b"+OK\n").await?; - socket.flush().await?; - } - - // 1. Register channel first so subsequent writes are buffered - let (tx, mut rx) = mpsc::channel::>(repl_channel_capacity); - let sent = Arc::new(AtomicU64::new(0)); - let acked = Arc::new(AtomicU64::new(0)); - { - let mut reg = replicas.senders.lock().await; - reg.push(ReplicaHandle { - tx, - sent: Arc::clone(&sent), - acked: Arc::clone(&acked), - }); - replicas.count.store(reg.len(), Ordering::Relaxed); - } - - // 2. Take snapshot and send (writes since snapshot are in channel) - let snap_bytes = - rmp_serde::to_vec(&store.snapshot()).map_err(|e| std::io::Error::other(e.to_string()))?; - let len = snap_bytes.len() as u32; - socket.write_all(&len.to_le_bytes()).await?; - socket.write_all(&snap_bytes).await?; - socket.flush().await?; - - // 3. Stream buffered + ongoing writes, and read acknowledgements - // - // The socket is bidirectional but used to carry frames one way only, which - // left the primary unable to say how far behind a replica was. The replica - // now writes back a cumulative count of applied frames; `sent - acked` is - // the lag. Reading and writing are selected over so a replica that stops - // acknowledging cannot stall the write side, and vice versa. - let (mut rd, mut wr) = tokio::io::split(socket); - let mut ack_buf = [0u8; 8]; - loop { - tokio::select! { - frame = rx.recv() => { - let Some(bytes) = frame else { break }; - let len = bytes.len() as u32; - wr.write_all(&len.to_le_bytes()).await?; - wr.write_all(&bytes).await?; - wr.flush().await?; - } - res = rd.read_exact(&mut ack_buf) => { - // A replica that closes its read side, or one running a build - // that predates acks, simply stops updating the gauge — it is - // not an error, so the stream continues either way. - if res.is_err() { - break; - } - let applied = u64::from_le_bytes(ack_buf); - // Monotonic: a reordered or replayed ack must never walk the - // high-water mark backwards and report negative lag. - acked.fetch_max(applied, Ordering::Relaxed); - } - } - } - Ok(()) -} - -// ── Replication client (replica side) ──────────────────────────────────────── - -async fn run_repl_client( - primary_addr: String, - store: Arc, - state: Arc, - repl_password: Option, - failover_timeout_secs: Option, - tx: broadcast::Sender, - tls: Option<(TlsConnector, String)>, -) { - let mut backoff_secs = 2u64; - let mut unreachable_since: Option = None; - - loop { - // Stop if already promoted (manual REPLICAOF NO ONE or earlier auto-promotion). - if !state.is_replica() { - return; - } - - info!("Replica: connecting to primary at {}", primary_addr); - match TcpStream::connect(&primary_addr).await { - Err(e) => { - warn!("Replica: connect failed: {}", e); - unreachable_since.get_or_insert_with(std::time::Instant::now); - } - Ok(socket) => { - // Primary is reachable — reset the unreachable timer. - unreachable_since = None; - backoff_secs = 2; - - // TLS is what makes the primary's *identity* checked, not just - // the channel encrypted: without it a DNS hijack or an on-path - // attacker can feed this replica an arbitrary keyspace, and the - // replica has no way to tell. - let result = match &tls { - None => { - sync_from_primary( - &mut { socket }, - &store, - repl_password.as_deref(), - &tx, - &state, - ) - .await - } - Some((connector, servername)) => { - match ServerName::try_from(servername.clone()) { - Err(_) => { - error!( - "Replica: '{}' is not a valid TLS server name — set \ - RECACHED_REPL_TLS_SERVERNAME to the name on the primary's \ - certificate", - servername - ); - return; - } - Ok(name) => { - match tokio::time::timeout( - handshake_timeout(), - connector.connect(name, socket), - ) - .await + }; + let s = Arc::clone(&store_tcp); + let t = tx_tcp.clone(); + let p = Arc::clone(&pass_tcp); + let ps = Arc::clone(&pubsub_tcp); + let wr = Arc::clone(&watch_tcp); + let tls = Arc::clone(&tls_tcp); + let sc = Arc::clone(&snap_tcp); + tokio::spawn(async move { + let _permit = permit; + if let Some(acc) = tls.as_ref() { + // Bounded: the permit is already held, so a peer + // that opens a socket and never negotiates would + // otherwise occupy a slot indefinitely. + match tokio::time::timeout(handshake_timeout(), acc.accept(socket)) + .await { - Err(_) => Err(std::io::Error::new( - ErrorKind::TimedOut, - "TLS handshake with primary timed out", - )), - Ok(Err(e)) => Err(std::io::Error::other(format!( - "TLS handshake with primary failed: {e} — check that the \ - primary has RECACHED_TLS_CERT set and that \ - RECACHED_REPL_TLS_CA trusts it" - ))), - Ok(Ok(mut stream)) => { - sync_from_primary( - &mut stream, - &store, - repl_password.as_deref(), - &tx, - &state, + Ok(Ok(tls_stream)) => { + handle_tcp( + tls_stream, + s, + t, + p, + ps, + wr, + sc, + addr.to_string(), ) .await } + Ok(Err(e)) => { + warn!("TCP TLS handshake failed from {}: {}", addr, e) + } + Err(_) => { + debug!("TCP TLS handshake from {} timed out", addr) + } } + } else { + handle_tcp(socket, s, t, p, ps, wr, sc, addr.to_string()).await; } - } + }); } - }; - - if let Err(e) = result { - warn!("Replica: sync ended: {}", e); - // Sync dropped — primary may be gone; start tracking if not already. - unreachable_since.get_or_insert_with(std::time::Instant::now); - } - } - } - - // Auto-failover: promote if primary has been unreachable long enough. - if let (Some(timeout), Some(since)) = (failover_timeout_secs, unreachable_since) { - let elapsed = since.elapsed().as_secs(); - if elapsed >= timeout { - warn!( - "Replica: primary unreachable for {}s (timeout {}s) — auto-promoting to primary", - elapsed, timeout - ); - state.promote_to_primary(); - return; - } - info!( - "Replica: primary unreachable for {}s / {}s before auto-failover", - elapsed, timeout - ); - } - - tokio::time::sleep(tokio::time::Duration::from_secs(backoff_secs)).await; - backoff_secs = (backoff_secs * 2).min(30); - } -} - -async fn sync_from_primary( - socket: &mut S, - store: &KeyValueStore, - repl_password: Option<&str>, - tx: &broadcast::Sender, - state: &ServerState, -) -> std::io::Result<()> -where - S: AsyncRead + AsyncWrite + Unpin, -{ - // 0. Send auth password if configured - if let Some(pwd) = repl_password { - let msg = format!("{}\n", pwd); - socket.write_all(msg.as_bytes()).await?; - socket.flush().await?; - // Read "+OK\n" (4 bytes) - let mut resp = [0u8; 4]; - socket.read_exact(&mut resp).await?; - if &resp != b"+OK\n" { - return Err(std::io::Error::new( - ErrorKind::PermissionDenied, - "replication auth rejected by primary", - )); - } - } - - // 1. Receive full snapshot - let mut len_buf = [0u8; 4]; - socket.read_exact(&mut len_buf).await?; - let snap_len = u32::from_le_bytes(len_buf) as usize; - if snap_len > MAX_REPL_FRAME_BYTES { - return Err(std::io::Error::new( - ErrorKind::InvalidData, - format!("snapshot frame too large ({snap_len} > {MAX_REPL_FRAME_BYTES} bytes)"), - )); - } - let mut snap_bytes = vec![0u8; snap_len]; - socket.read_exact(&mut snap_bytes).await?; - - match rmp_serde::from_slice::>(&snap_bytes) { - Ok(entries) => { - let count = entries.len(); - store.restore(entries); - info!("Replica: snapshot loaded ({} entries)", count); - } - Err(e) => { - return Err(std::io::Error::new(ErrorKind::InvalidData, e.to_string())); - } - } - - // 2. Stream write commands from primary, acknowledging what we apply - // - // Every frame is counted, including one that fails to parse: the primary - // counts frames it sent, so skipping a bad frame here would desynchronise - // the two offsets and understate lag forever after. - let mut applied: u64 = 0; - loop { - let mut len_buf = [0u8; 4]; - socket.read_exact(&mut len_buf).await?; - let cmd_len = u32::from_le_bytes(len_buf) as usize; - if cmd_len > MAX_REPL_FRAME_BYTES { - return Err(std::io::Error::new( - ErrorKind::InvalidData, - format!("command frame too large ({cmd_len} > {MAX_REPL_FRAME_BYTES} bytes)"), - )); - } - let mut cmd_bytes = vec![0u8; cmd_len]; - socket.read_exact(&mut cmd_bytes).await?; - applied += 1; - - match Value::parse(&cmd_bytes) { - Ok((value, _)) => { - // Replication frames are broadcast as RESP3 Push (>N\r\n); normalise to - // Array so Command::from_value can parse them. - let normalised = match value { - Value::Push(inner) => Value::Array(Some(inner)), - other => other, - }; - if let Ok(cmd) = Command::from_value(normalised) { - let keys = primary_keys(&cmd); - store.execute(cmd); - // Relay the applied write so this replica's own WebSocket - // clients see it, and any sub-replicas / AOF get it too - // (enables multi-tier replication and replica WS push). - let _ = tx.send(Arc::new(SyncPush { - origin: 0, - keys, - resp: cmd_bytes.clone(), - })); - state.on_write(&cmd_bytes).await; + Err(e) => warn!("TCP accept error: {}", e), } } - Err(e) => warn!("Replica: bad command from primary: {}", e), - } - - // Acknowledge on the same socket. TcpStream is unbuffered, so this is a - // single 8-byte write with no flush; a failure means the primary is - // gone, which the next read will surface with a better error. - if socket.write_all(&applied.to_le_bytes()).await.is_err() { - warn!("Replica: failed to send replication acknowledgement"); - } - } -} - -// ── security helpers ───────────────────────────────────────────────────────── - -/// Constant-time byte slice equality to prevent timing-based password leaks. -fn ct_eq_bytes(a: &[u8], b: &[u8]) -> bool { - if a.len() != b.len() { - return false; - } - a.iter() - .zip(b.iter()) - .fold(0u8, |acc, (x, y)| acc | (x ^ y)) - == 0 -} - -// ── sync scoping ───────────────────────────────────────────────────────────── - -/// One mutation pushed towards WebSocket peers: the RESP push frame plus the -/// keys it touches, so each connection can filter against its sync scopes -/// without re-parsing the frame. Wrapped in `Arc` — the broadcast channel -/// clones the payload once per receiver, so a clone is a refcount bump. -struct SyncPush { - origin: u64, - keys: Vec, - resp: Vec, -} - -type SyncMsg = Arc; - -/// True when a mutation touching `keys` is visible to a connection whose sync -/// scopes are `scopes`. A mutation with no keys (FLUSHDB) affects every scope. -fn scopes_match(scopes: &[String], keys: &[String]) -> bool { - keys.is_empty() - || keys - .iter() - .any(|k| scopes.iter().any(|p| core_engine::store::glob_match(p, k))) -} - -/// Verify a signed sync-scope token and return the granted patterns. -/// -/// Token format: `base64url(payload) "." base64url(hmac_sha256(secret, base64url(payload)))` -/// where payload is comma-separated glob patterns with an optional -/// `|` suffix. The HMAC is computed over the *encoded* -/// payload string, so minting in JS is one `createHmac` call on the base64url -/// text — no byte-level canonicalisation questions. -fn verify_sync_token(secret: &str, token: &str) -> Result, &'static str> { - use base64::Engine as _; - use hmac::{Hmac, Mac}; - use sha2::Sha256; - - let engine = base64::engine::general_purpose::URL_SAFE_NO_PAD; - let (payload_b64, sig_b64) = token.split_once('.').ok_or("malformed token")?; - let sig = engine.decode(sig_b64).map_err(|_| "malformed signature")?; - let mut mac = - Hmac::::new_from_slice(secret.as_bytes()).expect("HMAC accepts any key length"); - mac.update(payload_b64.as_bytes()); - if !ct_eq_bytes(&sig, &mac.finalize().into_bytes()) { - return Err("invalid signature"); - } - let payload_bytes = engine - .decode(payload_b64) - .map_err(|_| "malformed payload")?; - let payload = String::from_utf8(payload_bytes).map_err(|_| "malformed payload")?; - let (patterns_str, expiry) = match payload.split_once('|') { - Some((p, e)) => (p, Some(e)), - None => (payload.as_str(), None), - }; - if let Some(e) = expiry { - let exp: u64 = e.parse().map_err(|_| "malformed expiry")?; - let now = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_secs(); - if now >= exp { - return Err("token expired"); - } - } - let patterns: Vec = patterns_str - .split(',') - .map(str::trim) - .filter(|s| !s.is_empty()) - .map(String::from) - .collect(); - if patterns.is_empty() { - return Err("token grants no patterns"); - } - // Token patterns reach `glob_match` without passing through the command - // parser, so the cap `check_pattern` applies to `KEYS`/`SCAN`/`PSUBSCRIBE` - // has to be repeated here. These are matched once per key per write, so an - // over-long one is the most expensive place to put a pattern — and a - // compromised or careless minting service should not be able to. - if patterns - .iter() - .any(|p| p.len() > core_engine::store::MAX_PATTERN_BYTES) - { - return Err("token grants an over-long pattern"); - } - Ok(patterns) -} - -/// What a command touches, for scope enforcement on token-scoped WebSocket -/// connections. -#[derive(Debug)] -enum CommandScope { - /// No key access (PING, AUTH, MULTI, SYNC, pub/sub) — always allowed. - KeyLess, - /// Touches exactly these keys — every one must match a scope pattern. - Keys(Vec), - /// Keyspace-wide or administrative — denied on scoped connections. - Admin, -} - -fn command_scope(cmd: &Command) -> CommandScope { - match cmd { - Command::Ping(_) - | Command::Auth(_) - | Command::Hello(_) - | Command::Multi - | Command::Exec - | Command::Discard - | Command::Subscribe(_) - | Command::Unsubscribe(_) - | Command::PSubscribe(_) - | Command::PUnsubscribe(_) - | Command::Publish(_, _) - | Command::Sync(_) - // QSUB patterns are scope-checked against the grant in the WS handler. - | Command::QSub(_) - | Command::QUnsub(_) - // QUIT and CLIENT describe the connection itself, which a scoped - // connection is entitled to know about; COMMAND describes the server's - // vocabulary, which is public. - | Command::Quit - | Command::Client(_) - | Command::CommandQuery(_) - // CLUSTER and MODULE answer the same sentence to everyone — "not a - // cluster", "no modules" — and describe no state a scope could protect. - | Command::Cluster(_) - | Command::Module(_) - // Every MEMORY subcommand other than USAGE is refused outright, so - // there is nothing here to scope either. USAGE reads a key and is - // classified with the key commands below. - | Command::Memory(_) - | Command::Unknown(_) => CommandScope::KeyLess, - - Command::Keys(_) - | Command::Scan(_, _, _) - | Command::DbSize - | Command::FlushDb - | Command::Save - | Command::BgSave - | Command::LastSave - // INFO reports server-wide state — uptime, client counts, keyspace - // size, replication topology. A connection scoped to a handful of keys - // has no business reading it. - | Command::Info(_) - // CONFIG reports server-wide limits and whether auth is on. Same - // reasoning as INFO: not for a connection scoped to a few keys. - | Command::Config(_) - // PUBSUB enumerates every channel every other client is subscribed to. - // A scoped connection can already SUBSCRIBE to any channel it can name - // — channels are outside the scope system entirely — but naming and - // listing are different powers, the same way GET is scoped and KEYS is - // Admin. NUMSUB and NUMPAT ride along rather than splitting the family - // across two scopes for one subcommand's worth of difference. - | Command::PubSub(_) - | Command::ReplicaOfNoOne => CommandScope::Admin, - - Command::ESet(k, _) - | Command::Set(k, _, _) - | Command::Get(k) - | Command::Append(k, _) - | Command::Strlen(k) - | Command::GetRange(k, _, _) - | Command::GetSet(k, _) - | Command::SetNx(k, _) - | Command::SetEx(k, _, _) - | Command::PSetEx(k, _, _) - | Command::Incr(k) - | Command::Decr(k) - | Command::IncrBy(k, _) - | Command::DecrBy(k, _) - | Command::Expire(k, _) - | Command::PExpire(k, _) - | Command::ExpireAt(k, _) - | Command::PExpireAt(k, _) - | Command::Ttl(k) - | Command::PTtl(k) - | Command::Persist(k) - | Command::Type(k) - | Command::MemoryUsage(k) - | Command::HSet(k, _) - | Command::HGet(k, _) - | Command::HGetAll(k) - | Command::HDel(k, _) - | Command::HKeys(k) - | Command::HVals(k) - | Command::HLen(k) - | Command::HIncrBy(k, _, _) - | Command::HIncrByFloat(k, _, _) - | Command::HExists(k, _) - | Command::HSetNx(k, _, _) - | Command::HMGet(k, _) - | Command::HScan(k, _) - | Command::SScan(k, _) - | Command::ZScan(k, _) - | Command::LPush(k, _) - | Command::RPush(k, _) - | Command::LPushX(k, _) - | Command::RPushX(k, _) - | Command::LPop(k, _) - | Command::RPop(k, _) - | Command::LRange(k, _, _) - | Command::LLen(k) - | Command::LIndex(k, _) - | Command::LSet(k, _, _) - | Command::LRem(k, _, _) - | Command::LTrim(k, _, _) - | Command::SAdd(k, _) - | Command::SMembers(k) - | Command::SRem(k, _) - | Command::SCard(k) - | Command::SIsMember(k, _) - | Command::SMIsMember(k, _) - | Command::SPop(k, _) - | Command::SRandMember(k, _) - | Command::ZAdd(k, _, _) - | Command::ZRange(k, _, _, _) - | Command::ZRevRange(k, _, _, _) - | Command::ZRangeByScore(k, _, _, _, _) - | Command::ZRevRangeByScore(k, _, _, _, _) - | Command::ZScore(k, _) - | Command::ZMScore(k, _) - | Command::ZRank(k, _) - | Command::ZRevRank(k, _) - | Command::ZRem(k, _) - | Command::ZCard(k) - | Command::ZIncrBy(k, _, _) - | Command::ZCount(k, _, _) - | Command::RlSet(k, _, _) - | Command::RlCheck(k, _) - | Command::JSet(k, _, _) - | Command::JGet(k, _) - | Command::JMerge(k, _) => CommandScope::Keys(vec![k.clone()]), - - Command::Del(keys) - | Command::Unlink(keys) - | Command::MGet(keys) - | Command::Exists(keys) - | Command::SInter(keys) - | Command::SUnion(keys) - | Command::SDiff(keys) - | Command::Watch(keys) - | Command::Unwatch(keys) => CommandScope::Keys(keys.clone()), - - Command::MSet(pairs) => CommandScope::Keys(pairs.iter().map(|(k, _)| k.clone()).collect()), - Command::Rename(src, dst) | Command::SMove(src, dst, _) => { - CommandScope::Keys(vec![src.clone(), dst.clone()]) - } - Command::SInterStore(dst, keys) - | Command::SUnionStore(dst, keys) - | Command::SDiffStore(dst, keys) => { - let mut all = keys.clone(); - all.push(dst.clone()); - CommandScope::Keys(all) - } - - // Scope enforcement applies to the wrapped command. - Command::Dedup(_, _, inner) => command_scope(inner), + }); } -} -/// Handle the SYNC command for one WebSocket connection, returning the RESP -/// reply. Forms: -/// `SYNC` — list this connection's current scopes -/// `SYNC TOKEN ` — set scopes from a signed token (requires -/// `RECACHED_SYNC_SECRET` on the server) -/// `SYNC [...]` — set scopes directly (only allowed when no -/// secret is configured — a bandwidth filter, not -/// an authorization boundary) -fn handle_sync_command( - args: &[String], - secret: Option<&str>, - scopes: &mut Option>, - conn_id: u64, -) -> Vec { - fn patterns_reply(patterns: &[String]) -> Vec { - Value::Array(Some( - patterns - .iter() - .map(|p| Value::BulkString(Some(p.clone().into_bytes()))) - .collect(), - )) - .serialize() - } - match args { - [] => patterns_reply(scopes.as_deref().unwrap_or(&[])), - [kw, token] if kw.eq_ignore_ascii_case("token") => { - let Some(secret) = secret else { - return b"-ERR SYNC TOKEN requires RECACHED_SYNC_SECRET to be configured on the server\r\n" - .to_vec(); - }; - match verify_sync_token(secret, token) { - Ok(patterns) => { - info!("WS conn {} scoped via token: {:?}", conn_id, patterns); - let reply = patterns_reply(&patterns); - *scopes = Some(patterns); - reply - } - Err(e) => Value::Error(format!("ERR invalid sync token: {}", e)).serialize(), - } - } - patterns => { - if secret.is_some() { - return b"-ERR this server requires signed scopes: use SYNC TOKEN \r\n" - .to_vec(); - } - let pats: Vec = patterns.iter().filter(|p| !p.is_empty()).cloned().collect(); - if pats.is_empty() { - return b"-ERR SYNC requires at least one pattern\r\n".to_vec(); + // ── graceful shutdown via oneshot channel ──────────────────────────── + let (shutdown_tx, mut shutdown_rx) = tokio::sync::oneshot::channel::<()>(); + tokio::spawn(async move { + #[cfg(unix)] + { + let mut sigterm = + tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) + .expect("failed to register SIGTERM handler"); + tokio::select! { + _ = tokio::signal::ctrl_c() => {}, + _ = sigterm.recv() => {}, } - info!("WS conn {} sync scopes set: {:?}", conn_id, pats); - let reply = patterns_reply(&pats); - *scopes = Some(pats); - reply - } - } -} - -// ── connection identity ────────────────────────────────────────────────────── - -// TCP mutation broadcasts use id=0; WS/TCP pubsub connections get ids ≥ 1. -static NEXT_CONN_ID: AtomicU64 = AtomicU64::new(1); - -fn next_conn_id() -> u64 { - NEXT_CONN_ID.fetch_add(1, Ordering::Relaxed) -} - -// ── pub/sub ─────────────────────────────────────────────────────────────────── - -enum PubSubMsg { - Message { - channel: String, - message: Vec, - }, - PMessage { - pattern: String, - channel: String, - message: Vec, - }, -} - -type PubSubSender = mpsc::UnboundedSender; - -struct PubSubHub { - channel_subs: HashMap>, - pattern_subs: Vec<(String, u64, PubSubSender)>, -} - -impl PubSubHub { - fn new() -> Self { - Self { - channel_subs: HashMap::new(), - pattern_subs: Vec::new(), } - } - - fn subscribe(&mut self, conn_id: u64, channel: &str, tx: PubSubSender) { - self.channel_subs - .entry(channel.to_string()) - .or_default() - .push((conn_id, tx)); - } - - fn psubscribe(&mut self, conn_id: u64, pattern: &str, tx: PubSubSender) { - self.pattern_subs.push((pattern.to_string(), conn_id, tx)); - } - - fn unsubscribe(&mut self, conn_id: u64, channel: &str) { - if let Some(v) = self.channel_subs.get_mut(channel) { - v.retain(|(id, _)| *id != conn_id); - if v.is_empty() { - self.channel_subs.remove(channel); - } + #[cfg(not(unix))] + { + let _ = tokio::signal::ctrl_c().await; } - } - - fn punsubscribe(&mut self, conn_id: u64, pattern: &str) { - self.pattern_subs - .retain(|(p, id, _)| !(p == pattern && *id == conn_id)); - } - - fn unsubscribe_all(&mut self, conn_id: u64) { - self.channel_subs.retain(|_, v| { - v.retain(|(id, _)| *id != conn_id); - !v.is_empty() - }); - self.pattern_subs.retain(|(_, id, _)| *id != conn_id); - } - - /// Channels with at least one live subscriber, for `PUBSUB CHANNELS`. - /// - /// `unsubscribe` and `unsubscribe_all` remove a channel's entry once its - /// last subscriber leaves, and `publish` drops senders whose receiver has - /// closed, so a key in `channel_subs` implies a live subscriber. The - /// `is_empty` guard covers the one window where it does not: a connection - /// that died between the last publish and its close handler. - fn active_channels(&self) -> impl Iterator { - self.channel_subs - .iter() - .filter(|(_, subs)| !subs.is_empty()) - .map(|(channel, _)| channel) - } - - /// Subscribers to one exact channel, for `PUBSUB NUMSUB`. Pattern - /// subscribers are deliberately not counted, matching Redis: a `PSUBSCRIBE` - /// is reported by `NUMPAT`, and counting it here would double-count a - /// client that holds both. - fn subscriber_count(&self, channel: &str) -> i64 { - self.channel_subs - .get(channel) - .map(|subs| subs.len() as i64) - .unwrap_or(0) - } - - /// Distinct patterns under subscription, for `PUBSUB NUMPAT`. Distinct is - /// the Redis definition: two clients on `news.*` are one pattern, not two. - fn pattern_count(&self) -> i64 { - let mut seen: Vec<&str> = self - .pattern_subs - .iter() - .map(|(p, _, _)| p.as_str()) - .collect(); - seen.sort_unstable(); - seen.dedup(); - seen.len() as i64 - } + let _ = shutdown_tx.send(()); + }); - /// Deliver to all matching subscribers; returns the count delivered. - fn publish(&mut self, channel: &str, message: &[u8]) -> i64 { - let mut count = 0i64; + loop { + tokio::select! { + biased; - if let std::collections::hash_map::Entry::Occupied(mut e) = - self.channel_subs.entry(channel.to_string()) - { - let subs = e.get_mut(); - subs.retain(|(_, tx)| { - let ok = tx - .send(PubSubMsg::Message { - channel: channel.to_string(), - message: message.to_vec(), - }) - .is_ok(); - if ok { - count += 1; + res = ws_listener.accept() => { + match res { + Ok((socket, addr)) => { + let _ = socket.set_nodelay(true); + if let Some(allowed) = &allowed_ips + && !allowed.contains(&addr.ip()) + { + debug!("WS: rejected IP {}", addr.ip()); + continue; + } + let permit = match Arc::clone(&semaphore).try_acquire_owned() { + Ok(p) => p, + Err(_) => { + warn!("WS: connection limit reached, dropping {}", addr); + continue; + } + }; + let s = Arc::clone(&store); + let t = tx.clone(); + let p = Arc::clone(&global_password); + let ps = Arc::clone(&pubsub); + let wr = Arc::clone(&watch_registry); + let tls = Arc::clone(&tls_acceptor); + let sc = Arc::clone(&state); + let ss = Arc::clone(&sync_secret); + let ao = Arc::clone(&allowed_origins); + let id = next_conn_id(); + tokio::spawn(async move { + let _permit = permit; + if let Some(acc) = tls.as_ref() { + match tokio::time::timeout(handshake_timeout(), acc.accept(socket)).await { + Ok(Ok(tls_stream)) => { + handle_ws(tls_stream, s, t, p, id, ps, wr, sc, ss, ao, addr.to_string()).await + } + Ok(Err(e)) => warn!("WS TLS handshake failed from {}: {}", addr, e), + Err(_) => debug!("WS TLS handshake from {} timed out", addr), + } + } else { + handle_ws(socket, s, t, p, id, ps, wr, sc, ss, ao, addr.to_string()).await; + } + }); + } + Err(e) => warn!("WS accept error: {}", e), } - ok - }); - if subs.is_empty() { - e.remove(); } - } - let pattern_txs: Vec<(String, PubSubSender)> = self - .pattern_subs - .iter() - .filter(|(p, _, _)| core_engine::store::glob_match(p, channel)) - .map(|(p, _, tx)| (p.clone(), tx.clone())) - .collect(); - for (pattern, tx) in pattern_txs { - if tx - .send(PubSubMsg::PMessage { - pattern, - channel: channel.to_string(), - message: message.to_vec(), - }) - .is_ok() - { - count += 1; + _ = &mut shutdown_rx => { + info!("Shutdown signal received, saving final snapshot..."); + state.save(&store).await; + info!("Done. Goodbye."); + break; } } - self.pattern_subs.retain(|(_, _, tx)| !tx.is_closed()); - count - } -} - -type SharedPubSub = Arc>; - -// ── observable keys ─────────────────────────────────────────────────────────── - -type WatchNotif = (String, Value); -type WatchMap = HashMap)>>; - -/// Watched-key and live-query registry. `watched_keys` / `watched_patterns` -/// mirror the map lengths (updated by every writer while holding the lock) so -/// the per-write hot path can skip the mutexes entirely when nothing is -/// watched. -struct WatchHub { - /// Exact-key watchers (WATCH). - map: tokio::sync::Mutex, - watched_keys: AtomicUsize, - /// Glob-pattern subscribers (QSUB live queries), keyed by pattern. - patterns: tokio::sync::Mutex, - watched_patterns: AtomicUsize, -} - -impl WatchHub { - fn new() -> WatchRegistry { - Arc::new(WatchHub { - map: tokio::sync::Mutex::new(HashMap::new()), - watched_keys: AtomicUsize::new(0), - patterns: tokio::sync::Mutex::new(HashMap::new()), - watched_patterns: AtomicUsize::new(0), - }) - } - - fn is_empty(&self) -> bool { - self.watched_keys.load(Ordering::Relaxed) == 0 - && self.watched_patterns.load(Ordering::Relaxed) == 0 } - /// Call after mutating the key map, while still holding the lock. - fn sync_len(&self, map: &WatchMap) { - self.watched_keys.store(map.len(), Ordering::Relaxed); - } - - /// Call after mutating the pattern map, while still holding the lock. - fn sync_patterns_len(&self, map: &WatchMap) { - self.watched_patterns.store(map.len(), Ordering::Relaxed); - } + Ok(()) } -type WatchRegistry = Arc; - -/// Extract the key(s) that `cmd` writes to, without inspecting the response. -/// Used together with `broadcast_for()` — only call this when `broadcast_for` -/// already confirmed a mutation occurred. -fn primary_keys(cmd: &Command) -> Vec { - match cmd { - Command::ESet(k, _) - | Command::Set(k, _, _) - | Command::Append(k, _) - | Command::GetSet(k, _) - | Command::SetNx(k, _) - | Command::SetEx(k, _, _) - | Command::PSetEx(k, _, _) - | Command::Incr(k) - | Command::Decr(k) - | Command::IncrBy(k, _) - | Command::DecrBy(k, _) - | Command::Expire(k, _) - | Command::PExpire(k, _) - | Command::ExpireAt(k, _) - | Command::PExpireAt(k, _) - | Command::Persist(k) - | Command::HSet(k, _) - | Command::HDel(k, _) - | Command::HSetNx(k, _, _) - | Command::HIncrBy(k, _, _) - | Command::HIncrByFloat(k, _, _) - | Command::LPush(k, _) - | Command::RPush(k, _) - | Command::LPushX(k, _) - | Command::RPushX(k, _) - | Command::LPop(k, _) - | Command::RPop(k, _) - | Command::LSet(k, _, _) - | Command::LRem(k, _, _) - | Command::LTrim(k, _, _) - | Command::SAdd(k, _) - | Command::SRem(k, _) - | Command::SPop(k, _) - | Command::SInterStore(k, _) - | Command::SUnionStore(k, _) - | Command::SDiffStore(k, _) - | Command::ZAdd(k, _, _) - | Command::ZRem(k, _) - | Command::ZIncrBy(k, _, _) - | Command::RlSet(k, _, _) - | Command::JSet(k, _, _) - | Command::JMerge(k, _) => vec![k.clone()], - Command::Del(keys) | Command::Unlink(keys) => keys.clone(), - Command::MSet(pairs) => pairs.iter().map(|(k, _)| k.clone()).collect(), - Command::Rename(src, dst) | Command::SMove(src, dst, _) => { - vec![src.clone(), dst.clone()] - } - _ => vec![], - } -} - -fn encode_keychange(key: &str, value: &Value) -> Vec { - Value::Array(Some(vec![ - Value::BulkString(Some(b"keychange".to_vec())), - Value::BulkString(Some(key.as_bytes().to_vec())), - value.clone(), - ])) - .serialize() -} - -/// Push keychange notifications for a *confirmed* mutation. Callers must have -/// already established that `cmd` mutated the store (via `broadcast_for`). -async fn notify_watchers(registry: &WatchRegistry, cmd: &Command, store: &KeyValueStore) { - if registry.is_empty() { - return; - } - let keys = primary_keys(cmd); - if keys.is_empty() { - return; - } - // Fetch current values from DashMap *before* acquiring the registry lock - // to avoid holding two locks simultaneously. - let key_values: Vec<(String, Value)> = keys - .iter() - .map(|k| (k.clone(), store.get_current(k))) - .collect(); - if registry.watched_keys.load(Ordering::Relaxed) > 0 { - let mut reg = registry.map.lock().await; - for (key, value) in &key_values { - if let Some(subs) = reg.get_mut(key) { - subs.retain(|(_, tx)| tx.send((key.clone(), value.clone())).is_ok()); - if subs.is_empty() { - reg.remove(key); - } - } - } - registry.sync_len(®); - } - // Live queries: any registered glob pattern matching a touched key gets - // the same keychange notification. - if registry.watched_patterns.load(Ordering::Relaxed) > 0 { - let mut pats = registry.patterns.lock().await; - let mut emptied = false; - for (pattern, subs) in pats.iter_mut() { - for (key, value) in &key_values { - if core_engine::store::glob_match(pattern, key) { - subs.retain(|(_, tx)| tx.send((key.clone(), value.clone())).is_ok()); - } - } - emptied |= subs.is_empty(); - } - if emptied { - pats.retain(|_, subs| !subs.is_empty()); - } - registry.sync_patterns_len(&pats); - } -} - -/// Announce a `FLUSHDB` to live queries. -/// -/// Emitting a keychange per deleted key would mean one frame per key in the -/// keyspace — potentially millions — for a single command. Instead each -/// registered pattern receives one sentinel, delivered as a keychange whose key -/// is the pattern and whose value is nil. Subscribers treat it as "every key -/// matching this pattern is gone", which is exactly what happened, at O(patterns) -/// instead of O(keys). -/// -/// Explicitly `WATCH`ed keys are notified individually — that set is bounded by -/// the connection limit and callers expect per-key precision there. -async fn notify_flushdb(registry: &WatchRegistry, watched_before: Vec) { - if registry.watched_keys.load(Ordering::Relaxed) > 0 && !watched_before.is_empty() { - let mut reg = registry.map.lock().await; - for key in &watched_before { - if let Some(subs) = reg.get_mut(key) { - subs.retain(|(_, tx)| tx.send((key.clone(), Value::BulkString(None))).is_ok()); - } - } - registry.sync_len(®); - } - if registry.watched_patterns.load(Ordering::Relaxed) > 0 { - let mut pats = registry.patterns.lock().await; - let mut emptied = false; - for (pattern, subs) in pats.iter_mut() { - let sentinel = pattern.clone(); - subs.retain(|(_, tx)| tx.send((sentinel.clone(), Value::BulkString(None))).is_ok()); - emptied |= subs.is_empty(); - } - if emptied { - pats.retain(|_, subs| !subs.is_empty()); - } - registry.sync_patterns_len(&pats); - } -} - -/// Drop all of `conn_id`'s live-query subscriptions. Called on QUNSUB (all -/// form) and on connection close. -async fn unregister_all_qsubs( - registry: &WatchRegistry, - conn_id: u64, - qsub_patterns: &mut HashSet, -) { - if qsub_patterns.is_empty() { - return; - } - let mut pats = registry.patterns.lock().await; - for p in qsub_patterns.drain() { - if let Some(subs) = pats.get_mut(&p) { - subs.retain(|(id, _)| *id != conn_id); - if subs.is_empty() { - pats.remove(&p); - } - } - } - registry.sync_patterns_len(&pats); -} - -/// Post-write fan-out shared by the TCP and WS command paths: WebSocket sync -/// broadcast, AOF/replication log, and watch notifications. Structured so that -/// with no WS clients, no replicas, no AOF, and no watched keys — the common -/// standalone-server case — a write costs zero locks and zero allocations here. -async fn apply_write_effects( - cmd: &Command, - response: &Value, - tx: &broadcast::Sender, - origin: u64, - state: &ServerState, - watch_registry: &WatchRegistry, - store: &KeyValueStore, -) { - let has_ws = tx.receiver_count() > 0; - let needs_log = state.needs_write_log(); - let has_watch = !watch_registry.is_empty(); - if !has_ws && !needs_log && !has_watch { - return; - } - let Some(msg) = broadcast_for(cmd, response) else { - return; - }; - if needs_log { - state.on_write(&msg).await; - } - if has_watch { - if matches!(cmd, Command::FlushDb) { - // primary_keys() is empty for FLUSHDB, so the generic notifier has - // nothing to announce — subscribers would silently miss the wipe. - let watched: Vec = { - let reg = watch_registry.map.lock().await; - reg.keys().cloned().collect() - }; - notify_flushdb(watch_registry, watched).await; - } else { - notify_watchers(watch_registry, cmd, store).await; - } - } - if has_ws { - let _ = tx.send(Arc::new(SyncPush { - origin, - keys: primary_keys(cmd), - resp: msg, - })); - } -} - -/// Drop all of `conn_id`'s WATCH registrations and clear `watched_keys`. -/// Called at every transaction boundary (EXEC, DISCARD) and on connection close, -/// matching Redis semantics that WATCH state is flushed by EXEC/DISCARD. -async fn unregister_all_watches( - registry: &WatchRegistry, - conn_id: u64, - watched_keys: &mut HashSet, -) { - if watched_keys.is_empty() { - return; - } - let mut reg = registry.map.lock().await; - for key in watched_keys.drain() { - if let Some(subs) = reg.get_mut(&key) { - subs.retain(|(id, _)| *id != conn_id); - if subs.is_empty() { - reg.remove(&key); - } - } - } - registry.sync_len(®); -} - -// ── helpers ────────────────────────────────────────────────────────────────── - -/// Encode a pub/sub delivery for a connection speaking protocol `protover`. -/// -/// RESP2 has no push type, so a subscribed RESP2 client expects a plain array -/// and cannot parse a `>` frame at all. RESP3 clients want the push type so -/// deliveries are distinguishable from command replies on a multiplexed -/// connection. The WebSocket transport is RESP3 by definition — the sync -/// protocol is specified in terms of push frames — and passes 3. -/// Handle `HELLO [protover]`, updating `protover` in place on success. -/// -/// Returns the serialized reply. An unsupported version leaves the connection's -/// current protocol untouched and replies `-NOPROTO`, which is what lets a -/// client probe for RESP3 and fall back cleanly rather than being disconnected. -fn process_hello( - requested: Option<&str>, - protover: &mut u8, - is_authenticated: bool, - is_replica: bool, -) -> Vec { - if let Some(raw) = requested { - match raw.parse::() { - Ok(v @ (2 | 3)) => *protover = v, - _ => { - return Value::Error("NOPROTO unsupported protocol version".to_string()) - .serialize(); - } - } - } - - // Pre-auth HELLO reports the protocol but nothing about the server, so an - // unauthenticated client cannot use it to fingerprint the deployment. - if !is_authenticated { - return Value::Error("NOAUTH HELLO must be called with authentication".to_string()) - .serialize(); - } - - let fields = vec![ - ("server", Value::BulkString(Some(b"recached".to_vec()))), - ( - "version", - Value::BulkString(Some(env!("CARGO_PKG_VERSION").as_bytes().to_vec())), - ), - ("proto", Value::Integer(*protover as i64)), - ("mode", Value::BulkString(Some(b"standalone".to_vec()))), - ( - "role", - Value::BulkString(Some(if is_replica { - b"replica".to_vec() - } else { - b"master".to_vec() - })), - ), - ("modules", Value::Array(Some(vec![]))), - ]; - - if *protover >= 3 { - Value::Map( - fields - .into_iter() - .map(|(k, v)| (Value::BulkString(Some(k.as_bytes().to_vec())), v)) - .collect(), - ) - .serialize() - } else { - // RESP2 has no map type; Redis flattens to alternating key/value. - let mut flat = Vec::with_capacity(fields.len() * 2); - for (k, v) in fields { - flat.push(Value::BulkString(Some(k.as_bytes().to_vec()))); - flat.push(v); - } - Value::Array(Some(flat)).serialize() - } -} - -// ── INFO ───────────────────────────────────────────────────────────────────── - -/// Redis compatibility level advertised as `redis_version`. -/// -/// Clients feature-gate on this field, so it cannot be Recached's own version: -/// a library seeing `redis_version:0.2.3` concludes the server predates -/// everything and disables features it could safely use. 6.2 is the honest -/// floor — RESP3 and `HELLO` exist there, which Recached implements, while -/// nothing in 7.x that Recached lacks (functions, `OBJECT FREQ`, sharded -/// pub/sub) gets advertised. The real version ships alongside it as -/// `recached_version`, the same split KeyDB and Dragonfly use. -const REDIS_COMPAT_VERSION: &str = "6.2.0"; - -/// Sections `INFO` reports when called with no arguments. -const DEFAULT_INFO_SECTIONS: &[&str] = &[ - "server", - "clients", - "memory", - "persistence", - "stats", - "replication", - "cluster", - "keyspace", - "recached", -]; - -/// Process-wide startup facts, set once by `main`. -/// -/// Threaded through a static rather than the connection-handler signatures: -/// these values are immutable for the life of the process and needed only by -/// `INFO`, and the handlers already carry a long parameter list. Tests that -/// exercise `render_info` build their own `ServerFacts` and never touch this. -static SERVER_FACTS: std::sync::OnceLock = std::sync::OnceLock::new(); - -fn server_facts() -> &'static ServerFacts { - SERVER_FACTS.get_or_init(ServerFacts::default) -} - -/// Startup facts `INFO` reports that are fixed for the life of the process. -/// Captured in `main` once rather than re-read from the environment per call. -#[derive(Clone, Debug)] -struct ServerFacts { - start: SystemTime, - /// Random per-process identifier, as Redis reports it: 40 hex chars. - run_id: String, - tcp_port: u16, - ws_port: u16, - max_connections: usize, - tls_enabled: bool, - auth_enabled: bool, - aof_enabled: bool, -} - -impl Default for ServerFacts { - fn default() -> Self { - Self { - start: SystemTime::now(), - run_id: String::new(), - tcp_port: 6379, - ws_port: 6380, - max_connections: DEFAULT_MAX_CONNECTIONS, - tls_enabled: false, - auth_enabled: false, - aof_enabled: false, - } - } -} - -fn generate_run_id() -> String { - use rand::Rng; - let mut rng = rand::rng(); - (0..40) - .map(|_| std::char::from_digit(rng.random_range(0..16), 16).unwrap_or('0')) - .collect() -} - -/// Replication numbers, resolved by the caller. -/// -/// The registry's depth and lag accessors are async (they lock per-replica -/// queues), and `render_info` is a pure synchronous formatter so it stays -/// trivially testable — so the caller awaits them and passes the results in. -#[derive(Clone, Copy, Debug, Default)] -struct ReplInfo { - connected: usize, - queue_depth: usize, - lag_frames: u64, -} - -/// Last keyspace walk, refreshed every 5s by the metrics sampler. -/// -/// `keyspace_sample()` is O(keyspace). A monitoring agent polling `INFO` once a -/// second must not walk every key each time, so `INFO` reports the sample -/// instead. `u64::MAX` means "not sampled yet" — `INFO` then walks once itself, -/// which only happens in the first few seconds of uptime. -static SAMPLED_KEYS: AtomicU64 = AtomicU64::new(u64::MAX); -static SAMPLED_VOLATILE_KEYS: AtomicU64 = AtomicU64::new(u64::MAX); -static SAMPLED_MEMORY_BYTES: AtomicU64 = AtomicU64::new(u64::MAX); - -fn store_sampled_keyspace(sample: KeyspaceSample) { - SAMPLED_KEYS.store(sample.keys as u64, Ordering::Relaxed); - SAMPLED_VOLATILE_KEYS.store(sample.volatile_keys as u64, Ordering::Relaxed); - SAMPLED_MEMORY_BYTES.store(sample.memory_bytes as u64, Ordering::Relaxed); -} - -fn sampled_keyspace(store: &KeyValueStore) -> KeyspaceSample { - let keys = SAMPLED_KEYS.load(Ordering::Relaxed); - if keys == u64::MAX { - // Sampler has not run yet — walk once so the first INFO is not blank. - let sample = store.keyspace_sample(); - store_sampled_keyspace(sample); - return sample; - } - KeyspaceSample { - keys: keys as usize, - volatile_keys: SAMPLED_VOLATILE_KEYS.load(Ordering::Relaxed) as usize, - memory_bytes: SAMPLED_MEMORY_BYTES.load(Ordering::Relaxed) as usize, - } -} - -/// Render bytes the way Redis does for `*_human` fields. -fn human_bytes(bytes: u64) -> String { - const UNITS: [(&str, f64); 4] = [ - ("G", 1024.0 * 1024.0 * 1024.0), - ("M", 1024.0 * 1024.0), - ("K", 1024.0), - ("B", 1.0), - ]; - for (suffix, size) in UNITS { - if bytes as f64 >= size { - return if suffix == "B" { - format!("{}B", bytes) - } else { - format!("{:.2}{}", bytes as f64 / size, suffix) - }; - } - } - "0B".to_string() -} - -fn eviction_policy_name(policy: EvictionPolicy) -> &'static str { - match policy { - EvictionPolicy::NoEviction => "noeviction", - EvictionPolicy::AllKeysLru => "allkeys-lru", - EvictionPolicy::AllKeysRandom => "allkeys-random", - EvictionPolicy::VolatileLru => "volatile-lru", - EvictionPolicy::VolatileTtl => "volatile-ttl", - } -} - -// ── CLIENT / CONFIG / COMMAND ───────────────────────────────────────────────── - -/// Wrong-subcommand error in Redis's wording, which some clients match on. -fn unknown_subcommand(container: &str, sub: &str) -> Value { - Value::Error(format!( - "ERR Unknown subcommand or wrong number of arguments for '{sub}'. \ - Try {container} HELP." - )) -} - -/// Handle `CLIENT `, mutating this connection's `meta` in place. -/// -/// Returns `None` for subcommands Recached does not implement, so the caller -/// can answer with the standard error rather than this function inventing a -/// reply. `SETNAME`, `SETINFO` and the protocol version write through to the -/// registry, which is what makes them visible to another connection's -/// `CLIENT LIST`. -fn handle_client_command(args: &[String], meta: &mut ClientMeta) -> Value { - let sub = args[0].to_uppercase(); - match (sub.as_str(), args.len()) { - ("ID", 1) => Value::Integer(meta.id as i64), - ("INFO", 1) => Value::BulkString(Some(meta.render().into_bytes())), - ("LIST", 1) => Value::BulkString(Some(client_list_lines().into_bytes())), - ("GETNAME", 1) => { - if meta.name.is_empty() { - Value::BulkString(None) - } else { - Value::BulkString(Some(meta.name.clone().into_bytes())) - } - } - ("SETNAME", 2) => { - // Redis reserves spaces and newlines because the name is echoed - // into the space-separated CLIENT LIST format. - if args[1].contains(' ') || args[1].contains('\n') { - return Value::Error( - "ERR Client names cannot contain spaces, newlines or special characters." - .to_string(), - ); - } - meta.name = args[1].clone(); - publish_client(meta.clone()); - Value::SimpleString("OK".to_string()) - } - ("SETINFO", 3) => match args[1].to_uppercase().as_str() { - "LIB-NAME" => { - meta.lib_name = args[2].clone(); - publish_client(meta.clone()); - Value::SimpleString("OK".to_string()) - } - "LIB-VER" => { - meta.lib_ver = args[2].clone(); - publish_client(meta.clone()); - Value::SimpleString("OK".to_string()) - } - other => Value::Error(format!("ERR Unrecognized option '{other}'")), - }, - ("HELP", 1) => Value::Array(Some( - [ - "CLIENT ", - "ID -- Return this connection's identifier.", - "INFO -- Return information about this connection.", - "LIST -- Return information about all connections.", - "GETNAME -- Return this connection's name.", - "SETNAME -- Set this connection's name.", - "SETINFO -- Identify the client library.", - ] - .iter() - .map(|l| Value::SimpleString(l.to_string())) - .collect(), - )), - // KILL, UNPAUSE, NO-EVICT and friends are administrative operations - // with real semantics. Answering +OK without performing them would be - // worse than saying no: a client would believe a connection had been - // killed or eviction disabled when nothing happened. - _ => unknown_subcommand("CLIENT", &args.join(" ")), - } -} - -/// The configuration parameters `CONFIG GET` reports, resolved from the values -/// actually in force rather than from a table of defaults. -fn config_parameters(facts: &ServerFacts, store: &KeyValueStore) -> Vec<(&'static str, String)> { - vec![ - ( - "maxmemory", - store.max_memory_bytes().unwrap_or(0).to_string(), - ), - ( - "maxmemory-policy", - eviction_policy_name(store.eviction_policy()).to_string(), - ), - ("maxclients", facts.max_connections.to_string()), - ("port", facts.tcp_port.to_string()), - ( - "tls-port", - if facts.tls_enabled { - facts.tcp_port.to_string() - } else { - "0".to_string() - }, - ), - ( - "appendonly", - if facts.aof_enabled { - "yes".into() - } else { - "no".into() - }, - ), - // Recached has a single keyspace. Clients that SELECT anything other - // than 0 need to know that before they try. - ("databases", "1".to_string()), - // Reported as masked, exactly as Redis does: the presence of a - // password is not a secret, its value is. - ( - "requirepass", - if facts.auth_enabled { - "*".into() - } else { - String::new() - }, - ), - ("proto-max-bulk-len", MAX_BULK_STRING_BYTES.to_string()), - ("timeout", "0".to_string()), - ("save", String::new()), - ] -} - -/// Handle `CONFIG `. -fn handle_config_command(args: &[String], facts: &ServerFacts, store: &KeyValueStore) -> Value { - let sub = args[0].to_uppercase(); - match sub.as_str() { - "GET" if args.len() >= 2 => { - let params = config_parameters(facts, store); - let mut out = Vec::new(); - for (name, value) in ¶ms { - if args[1..].iter().any(|pat| glob_match(pat, name)) { - out.push(Value::BulkString(Some(name.as_bytes().to_vec()))); - out.push(Value::BulkString(Some(value.clone().into_bytes()))); - } - } - Value::Array(Some(out)) - } - // Recached reads its configuration from the environment at startup and - // holds it behind an `Arc` for the life of the process, so there is - // nothing a runtime SET could change. Saying so is better than - // returning OK and leaving the operator to discover later that the - // limit they set never applied. - "SET" if args.len() >= 3 => Value::Error(format!( - "ERR CONFIG SET is not supported: Recached is configured at startup. \ - Set '{}' through the environment and restart.", - args[1] - )), - "RESETSTAT" if args.len() == 1 => Value::Error( - "ERR CONFIG RESETSTAT is not supported: counters are exported to Prometheus, \ - where resetting them would break rate calculations." - .to_string(), - ), - _ => unknown_subcommand("CONFIG", &args.join(" ")), - } -} - -/// Handle `CLUSTER `. -/// -/// Recached does not cluster, and this reports that the way Redis does. A -/// `redis-server` that was not started in cluster mode does **not** answer -/// `CLUSTER INFO` with `cluster_enabled:0` — it rejects the whole `CLUSTER` -/// container with this exact sentence, and publishes the flag in `INFO`'s -/// `# Cluster` section instead. Copying the sentence rather than inventing a -/// slot map means a client's "am I clustered" branch takes the same path here -/// as against the server it was written for, and `ERR unknown command` (which -/// is what Recached said before) is the one answer that reads as "too old to -/// ask" rather than "not a cluster". -fn handle_cluster_command(_args: &[String]) -> Value { - Value::Error("ERR This instance has cluster support disabled".to_string()) -} - -/// Handle `MODULE `. -/// -/// There is no module API, so the loaded-module list is empty — which is a -/// real answer, and the same one a stock `redis-server` gives. `LOAD`, -/// `LOADEX` and `UNLOAD` are refused rather than answered `+OK`, because an -/// operator who believes a module loaded has a harder problem than one who -/// was told no. -fn handle_module_command(args: &[String]) -> Value { - match (args[0].to_uppercase().as_str(), args.len()) { - ("LIST", 1) => Value::Array(Some(vec![])), - ("HELP", 1) => Value::Array(Some( - [ - "MODULE ", - "LIST -- Return a list of loaded modules. Recached loads none.", - ] - .iter() - .map(|l| Value::SimpleString((*l).to_string())) - .collect(), - )), - _ => unknown_subcommand("MODULE", &args.join(" ")), - } -} - -/// Handle `PUBSUB [arg ...]` against the live subscriber hub. -/// -/// Recached has shipped `SUBSCRIBE`, `PSUBSCRIBE` and `PUBLISH` from the start -/// with no way to see any of it: `PUBLISH` returns a delivery count, and that -/// was the only observable. The hub already holds both registries, so these -/// three answers are a read of state that existed all along. -/// -/// `SHARDCHANNELS` and `SHARDNUMSUB` are refused rather than answered with the -/// empty array a standalone `redis-server` gives. This is a deliberate -/// divergence: Redis's empty array means "no shard channels are subscribed" on -/// a server where `SSUBSCRIBE` works, and a client reading it would reasonably -/// follow up with one. Recached has no `SSUBSCRIBE` or `SPUBLISH` at all, so -/// the honest answer is that the question does not apply here. -fn handle_pubsub_command(args: &[String], hub: &PubSubHub) -> Value { - match (args[0].to_uppercase().as_str(), args.len()) { - // No pattern means every active channel. Redis matches the pattern - // against channel names with the same globber it uses for keys, and so - // does this — `glob_match` is the one Recached already applies to - // PSUBSCRIBE, so a pattern selects here exactly what it would there. - ("CHANNELS", 1) => Value::Array(Some( - hub.active_channels() - .map(|c| Value::BulkString(Some(c.as_bytes().to_vec()))) - .collect(), - )), - ("CHANNELS", 2) => Value::Array(Some( - hub.active_channels() - .filter(|c| core_engine::store::glob_match(&args[1], c)) - .map(|c| Value::BulkString(Some(c.as_bytes().to_vec()))) - .collect(), - )), - // Flat [channel, count, channel, count, ...]. A channel nobody is - // subscribed to reports 0 rather than being dropped, so a caller that - // asked about N channels can index the reply by position. - ("NUMSUB", _) => { - let mut out = Vec::with_capacity((args.len() - 1) * 2); - for channel in &args[1..] { - out.push(Value::BulkString(Some(channel.as_bytes().to_vec()))); - out.push(Value::Integer(hub.subscriber_count(channel))); - } - Value::Array(Some(out)) - } - ("NUMPAT", 1) => Value::Integer(hub.pattern_count()), - ("HELP", 1) => Value::Array(Some( - [ - "PUBSUB ", - "CHANNELS [pattern] -- Return the currently active channels.", - "NUMSUB [channel ...] -- Return the subscriber count per channel.", - "NUMPAT -- Return the number of distinct subscribed patterns.", - ] - .iter() - .map(|l| Value::SimpleString((*l).to_string())) - .collect(), - )), - _ => unknown_subcommand("PUBSUB", &args.join(" ")), - } -} - -/// Handle `MEMORY ` for everything except `USAGE`, which is a key -/// read and goes to the store. -/// -/// `DOCTOR`, `STATS`, `PURGE` and `MALLOC-STATS` all describe an allocator -/// Recached does not manage — it holds Rust values in a `DashMap` and has no -/// arena to report on or free. Saying so beats a fabricated report. -fn handle_memory_command(args: &[String]) -> Value { - match (args[0].to_uppercase().as_str(), args.len()) { - ("HELP", 1) => Value::Array(Some( - [ - "MEMORY ", - "USAGE [SAMPLES ] -- Bytes held by one key. SAMPLES is accepted \ - and ignored: the estimate always covers every element.", - ] - .iter() - .map(|l| Value::SimpleString((*l).to_string())) - .collect(), - )), - ("DOCTOR" | "STATS" | "PURGE" | "MALLOC-STATS", 1) => Value::Error(format!( - "ERR MEMORY {} is not supported: Recached does not manage its own allocator, \ - so it has nothing to report or free. MEMORY USAGE and INFO memory are the \ - measurements it can make.", - args[0].to_uppercase() - )), - _ => unknown_subcommand("MEMORY", &args.join(" ")), - } -} - -/// `COMMAND INFO`'s per-command reply: name, arity, flags, key positions. -fn command_info_entry(spec: &catalog::CommandSpec) -> Value { - Value::Array(Some(vec![ - Value::BulkString(Some(spec.name.as_bytes().to_vec())), - Value::Integer(spec.arity as i64), - Value::Array(Some( - spec.flags - .iter() - .map(|f| Value::SimpleString((*f).to_string())) - .collect(), - )), - Value::Integer(spec.first_key as i64), - Value::Integer(spec.last_key as i64), - Value::Integer(spec.step as i64), - // ACL categories, tips, key specs and subcommands: Redis 7 appends - // four more elements here. Recached has no ACL system and no - // subcommand tree to describe, so it reports them empty rather than - // omitting them — a client indexing element 6 gets an empty list - // instead of an out-of-range error. - Value::Array(Some(vec![])), - Value::Array(Some(vec![])), - Value::Array(Some(vec![])), - Value::Array(Some(vec![])), - ])) -} - -/// `COMMAND DOCS`'s per-command reply. RESP2 clients see the same pairs as a -/// flat array, which is how Redis degrades a map on the older protocol. -fn command_docs_entry(spec: &catalog::CommandSpec, protover: u8) -> Value { - let fields = vec![ - ( - "summary", - Value::BulkString(Some(spec.summary.as_bytes().to_vec())), - ), - ("since", Value::BulkString(Some(b"1.0.0".to_vec()))), - ( - "group", - Value::BulkString(Some(spec.group.as_bytes().to_vec())), - ), - ("arity", Value::Integer(spec.arity as i64)), - ]; - map_or_flat(fields, protover) -} - -/// RESP3 sends a map; RESP2 has no map type and flattens to alternating -/// key/value entries. Same split `HELLO` already makes. -fn map_or_flat(fields: Vec<(&str, Value)>, protover: u8) -> Value { - if protover >= 3 { - Value::Map( - fields - .into_iter() - .map(|(k, v)| (Value::BulkString(Some(k.as_bytes().to_vec())), v)) - .collect(), - ) - } else { - let mut flat = Vec::with_capacity(fields.len() * 2); - for (k, v) in fields { - flat.push(Value::BulkString(Some(k.as_bytes().to_vec()))); - flat.push(v); - } - Value::Array(Some(flat)) - } -} - -/// Handle `COMMAND [subcommand]`. -fn handle_command_query(args: &[String], protover: u8) -> Value { - let Some(sub) = args.first() else { - // Bare COMMAND: the whole catalog, as COMMAND INFO entries. - return Value::Array(Some( - catalog::CATALOG.iter().map(command_info_entry).collect(), - )); - }; - match sub.to_uppercase().as_str() { - "COUNT" if args.len() == 1 => Value::Integer(catalog::CATALOG.len() as i64), - "LIST" if args.len() == 1 => Value::Array(Some( - catalog::CATALOG - .iter() - .map(|c| Value::BulkString(Some(c.name.as_bytes().to_vec()))) - .collect(), - )), - "INFO" => { - if args.len() == 1 { - return Value::Array(Some( - catalog::CATALOG.iter().map(command_info_entry).collect(), - )); - } - // A name the server does not have replies nil in its slot, so the - // reply stays positionally aligned with the request. - Value::Array(Some( - args[1..] - .iter() - .map(|n| match catalog::lookup(n) { - Some(spec) => command_info_entry(spec), - None => Value::Array(None), - }) - .collect(), - )) - } - "DOCS" => { - let specs: Vec<&catalog::CommandSpec> = if args.len() == 1 { - catalog::CATALOG.iter().collect() - } else { - args[1..] - .iter() - .filter_map(|n| catalog::lookup(n)) - .collect() - }; - // Unknown names are absent from the map rather than nil-filled: - // COMMAND DOCS is keyed by name, so there is no slot to align. - let fields: Vec<(&str, Value)> = specs - .iter() - .map(|s| (s.name, command_docs_entry(s, protover))) - .collect(); - map_or_flat(fields, protover) - } - _ => unknown_subcommand("COMMAND", &args.join(" ")), - } -} - -/// Build the `INFO` payload for `sections` (empty = the default set). -/// -/// The format is load-bearing: `# Section` header, `field:value` lines, CRLF -/// throughout, and a blank line between sections. Every Redis client and -/// monitoring agent parses exactly that shape, so it is covered by tests rather -/// than left to formatting drift. Unknown section names yield no output, which -/// is what Redis does. -#[allow(clippy::too_many_arguments)] -fn render_info( - sections: &[String], - facts: &ServerFacts, - store: &KeyValueStore, - sample: KeyspaceSample, - is_replica: bool, - repl: ReplInfo, - last_save: i64, - live_queries: u64, - watched_keys: u64, -) -> String { - let wanted: Vec<&str> = if sections.is_empty() - || sections - .iter() - .any(|s| s == "all" || s == "everything" || s == "default") - { - DEFAULT_INFO_SECTIONS.to_vec() - } else { - sections.iter().map(|s| s.as_str()).collect() - }; - - let uptime = facts - .start - .elapsed() - .map(|d| d.as_secs()) - .unwrap_or_default(); - let mut out = String::new(); - - for section in wanted { - let body = match section { - "server" => { - format!( - "redis_version:{}\r\n\ - recached_version:{}\r\n\ - redis_mode:standalone\r\n\ - os:{}\r\n\ - arch_bits:{}\r\n\ - process_id:{}\r\n\ - run_id:{}\r\n\ - tcp_port:{}\r\n\ - recached_ws_port:{}\r\n\ - recached_tls_enabled:{}\r\n\ - recached_auth_enabled:{}\r\n\ - uptime_in_seconds:{}\r\n\ - uptime_in_days:{}\r\n", - REDIS_COMPAT_VERSION, - env!("CARGO_PKG_VERSION"), - std::env::consts::OS, - usize::BITS, - std::process::id(), - facts.run_id, - facts.tcp_port, - facts.ws_port, - u8::from(facts.tls_enabled), - u8::from(facts.auth_enabled), - uptime, - uptime / 86_400, - ) - } - "clients" => { - // A negative count would mean the guard accounting is broken; - // clamp rather than emit a value no client can parse. - let active = STAT_CONNECTIONS_ACTIVE.load(Ordering::Relaxed).max(0); - format!( - "connected_clients:{}\r\n\ - maxclients:{}\r\n\ - blocked_clients:0\r\n", - active, facts.max_connections, - ) - } - "memory" => { - let used = sample.memory_bytes as u64; - let max = store.max_memory_bytes().unwrap_or(0) as u64; - format!( - "used_memory:{}\r\n\ - used_memory_human:{}\r\n\ - maxmemory:{}\r\n\ - maxmemory_human:{}\r\n\ - maxmemory_policy:{}\r\n\ - recached_max_keys:{}\r\n", - used, - human_bytes(used), - max, - human_bytes(max), - eviction_policy_name(store.eviction_policy()), - store.max_keys().unwrap_or(0), - ) - } - "persistence" => { - // `loading` is what a client's ready-check reads to decide the - // server can serve traffic. Recached loads its snapshot before - // it binds a listener, so a client that can reach us is never - // looking at a loading server: the answer is always 0. - format!( - "loading:0\r\n\ - rdb_changes_since_last_save:{}\r\n\ - rdb_last_save_time:{}\r\n\ - rdb_bgsave_in_progress:0\r\n\ - aof_enabled:{}\r\n", - store.dirty_count(), - last_save, - u8::from(facts.aof_enabled), - ) - } - "stats" => { - format!( - "total_connections_received:{}\r\n\ - total_commands_processed:{}\r\n\ - keyspace_hits:{}\r\n\ - keyspace_misses:{}\r\n\ - evicted_keys:{}\r\n", - STAT_CONNECTIONS_TOTAL.load(Ordering::Relaxed), - STAT_COMMANDS_TOTAL.load(Ordering::Relaxed), - STAT_KEYSPACE_HITS.load(Ordering::Relaxed), - STAT_KEYSPACE_MISSES.load(Ordering::Relaxed), - store.evicted_count(), - ) - } - "replication" => { - // Redis still spells these `slave`; tooling greps for exactly - // that, so the compatible spelling is authoritative and the - // `replica` names are emitted alongside it. - format!( - "role:{}\r\n\ - connected_slaves:{}\r\n\ - connected_replicas:{}\r\n\ - recached_replication_queue_depth:{}\r\n\ - recached_replication_lag_frames:{}\r\n", - if is_replica { "slave" } else { "master" }, - repl.connected, - repl.connected, - repl.queue_depth, - repl.lag_frames, - ) - } - "keyspace" => { - // Redis omits the db line entirely when the database is empty. - if sample.keys == 0 { - String::new() - } else { - format!( - "db0:keys={},expires={},avg_ttl=0\r\n", - sample.keys, sample.volatile_keys, - ) - } - } - // How a cluster-aware client actually learns it is talking to a - // single node. `CLUSTER INFO` is not that channel: a `redis-server` - // built for standalone answers it with an error, not with - // `cluster_enabled:0`, so this line is the only place the answer - // exists. Reporting it costs one line and stops a client from - // guessing. - "cluster" => "cluster_enabled:0\r\n".to_string(), - // Recached-specific: the live-query machinery has no Redis analogue, - // so it gets its own section rather than being smuggled into one. - "recached" => { - format!( - "live_queries:{}\r\n\ - watched_keys:{}\r\n", - live_queries, watched_keys, - ) - } - _ => continue, - }; - - let title = { - let mut c = section.chars(); - match c.next() { - Some(f) => f.to_uppercase().collect::() + c.as_str(), - None => continue, - } - }; - out.push_str(&format!("# {}\r\n{}\r\n", title, body)); - } - - out -} - -fn encode_pubsub_msg(msg: PubSubMsg, protover: u8) -> Vec { - let frame = |parts: Vec| { - if protover >= 3 { - Value::Push(parts) - } else { - Value::Array(Some(parts)) - } - }; - match msg { - PubSubMsg::Message { channel, message } => frame(vec![ - Value::BulkString(Some(b"message".to_vec())), - Value::BulkString(Some(channel.into_bytes())), - Value::BulkString(Some(message)), - ]) - .serialize(), - PubSubMsg::PMessage { - pattern, - channel, - message, - } => frame(vec![ - Value::BulkString(Some(b"pmessage".to_vec())), - Value::BulkString(Some(pattern.into_bytes())), - Value::BulkString(Some(channel.into_bytes())), - Value::BulkString(Some(message)), - ]) - .serialize(), - } -} - -fn resp_subscribe_ack(kind: &str, channel: &str, count: usize) -> Vec { - Value::Array(Some(vec![ - Value::BulkString(Some(kind.as_bytes().to_vec())), - Value::BulkString(Some(channel.as_bytes().to_vec())), - Value::Integer(count as i64), - ])) - .serialize() -} - -/// Encodes a list of string parts as a RESP3 Push frame for WebSocket fan-out. -/// Uses `>` prefix so clients can distinguish server-initiated pushes from command responses. -/// Build a RESP3 Push frame from raw byte arguments. -/// -/// Bytes rather than `&str` because these frames carry stored values, which may -/// be arbitrary binary. Building them as a `String` would have required a lossy -/// conversion — silently corrupting the replicated, AOF-logged and -/// browser-synced copy of a value the store itself holds faithfully. -fn resp_push(parts: &[&[u8]]) -> Vec { - let mut out = format!(">{}\r\n", parts.len()).into_bytes(); - for part in parts { - out.extend_from_slice(format!("${}\r\n", part.len()).as_bytes()); - out.extend_from_slice(part); - out.extend_from_slice(b"\r\n"); - } - out -} - -/// Returns the RESP-encoded mutation to broadcast to WebSocket peers, or `None` -/// if the command mutated nothing (read-only or conditional-and-failed). -fn broadcast_for(cmd: &Command, response: &Value) -> Option> { - match cmd { - // Replays as SET: a replica has no connection to scope the lifetime to, - // and the owning server broadcasts the DEL when the connection closes. - Command::ESet(k, v) => Some(resp_push(&[b"SET", k.as_bytes(), v.as_slice()])), - Command::Set(k, v, opts) => { - // Without GET: nil response means NX/XX condition failed — don't broadcast. - // With GET: nil means key didn't exist before, but SET still happened. - let set_happened = opts.get || !matches!(response, Value::BulkString(None)); - if !set_happened { - return None; - } - match &opts.expiry { - None => Some(resp_push(&[b"SET", k.as_bytes(), v.as_slice()])), - Some(SetExpiry::Ex(s)) => { - let px = s.saturating_mul(1000).to_string(); - Some(resp_push(&[ - b"SET", - k.as_bytes(), - v.as_slice(), - b"PX", - px.as_bytes(), - ])) - } - Some(SetExpiry::Px(ms)) => { - let ms_s = ms.to_string(); - Some(resp_push(&[ - b"SET", - k.as_bytes(), - v.as_slice(), - b"PX", - ms_s.as_bytes(), - ])) - } - Some(SetExpiry::Exat(ts)) => { - let pxat = ts.saturating_mul(1000).to_string(); - Some(resp_push(&[ - b"SET", - k.as_bytes(), - v.as_slice(), - b"PXAT", - pxat.as_bytes(), - ])) - } - Some(SetExpiry::Pxat(ts)) => { - let ts_s = ts.to_string(); - Some(resp_push(&[ - b"SET", - k.as_bytes(), - v.as_slice(), - b"PXAT", - ts_s.as_bytes(), - ])) - } - Some(SetExpiry::KeepTtl) => { - Some(resp_push(&[b"SET", k.as_bytes(), v.as_slice(), b"KEEPTTL"])) - } - } - } - Command::Del(keys) | Command::Unlink(keys) => { - let mut parts: Vec<&[u8]> = vec![b"DEL"]; - let key_refs: Vec<&[u8]> = keys.iter().map(|s| s.as_bytes()).collect(); - parts.extend_from_slice(&key_refs); - Some(resp_push(&parts)) - } - Command::MSet(pairs) => { - let mut parts: Vec<&[u8]> = vec![b"MSET"]; - let flat: Vec> = pairs - .iter() - .flat_map(|(k, v)| [k.as_bytes().to_vec(), v.clone()]) - .collect(); - let flat_refs: Vec<&[u8]> = flat.iter().map(|s| s.as_slice()).collect(); - parts.extend_from_slice(&flat_refs); - Some(resp_push(&parts)) - } - Command::SetNx(k, v) => match response { - Value::Integer(1) => Some(resp_push(&[b"SET", k.as_bytes(), v.as_slice()])), - _ => None, - }, - Command::SetEx(k, secs, v) => { - let px = secs.saturating_mul(1000).to_string(); - Some(resp_push(&[ - b"SET", - k.as_bytes(), - v.as_slice(), - b"PX", - px.as_bytes(), - ])) - } - Command::PSetEx(k, ms, v) => { - let ms_s = ms.to_string(); - Some(resp_push(&[ - b"SET", - k.as_bytes(), - v.as_slice(), - b"PX", - ms_s.as_bytes(), - ])) - } - Command::Append(k, v) => match response { - Value::Integer(_) => Some(resp_push(&[b"APPEND", k.as_bytes(), v.as_slice()])), - _ => None, - }, - Command::GetSet(k, v) => Some(resp_push(&[b"SET", k.as_bytes(), v.as_slice()])), - Command::Incr(k) | Command::Decr(k) => match response { - Value::Integer(n) => { - let s = n.to_string(); - Some(resp_push(&[b"SET", k.as_bytes(), s.as_bytes()])) - } - _ => None, - }, - Command::IncrBy(k, _) | Command::DecrBy(k, _) => match response { - Value::Integer(n) => { - let s = n.to_string(); - Some(resp_push(&[b"SET", k.as_bytes(), s.as_bytes()])) - } - _ => None, - }, - Command::Expire(k, secs) => match response { - Value::Integer(1) => { - let ms = secs.saturating_mul(1000).to_string(); - Some(resp_push(&[b"PEXPIRE", k.as_bytes(), ms.as_bytes()])) - } - _ => None, - }, - Command::PExpire(k, ms) => match response { - Value::Integer(1) => { - let ms_s = ms.to_string(); - Some(resp_push(&[b"PEXPIRE", k.as_bytes(), ms_s.as_bytes()])) - } - _ => None, - }, - Command::ExpireAt(k, ts) => match response { - Value::Integer(1) => { - let ts_ms = ts.saturating_mul(1000).to_string(); - Some(resp_push(&[b"PEXPIREAT", k.as_bytes(), ts_ms.as_bytes()])) - } - _ => None, - }, - Command::PExpireAt(k, ts) => match response { - Value::Integer(1) => { - let ts_s = ts.to_string(); - Some(resp_push(&[b"PEXPIREAT", k.as_bytes(), ts_s.as_bytes()])) - } - _ => None, - }, - Command::Persist(k) => match response { - Value::Integer(1) => Some(resp_push(&[b"PERSIST", k.as_bytes()])), - _ => None, - }, - Command::FlushDb => Some(resp_push(&[b"FLUSHDB"])), - Command::Rename(src, dst) => match response { - Value::Error(_) => None, - _ => Some(resp_push(&[b"RENAME", src.as_bytes(), dst.as_bytes()])), - }, - - // ── Hash ───────────────────────────────────────────────────────────── - Command::HSet(k, pairs) => { - let mut parts: Vec> = vec![b"HSET".to_vec(), k.as_bytes().to_vec()]; - for (f, v) in pairs { - parts.push(f.as_bytes().to_vec()); - parts.push(v.clone()); - } - let refs: Vec<&[u8]> = parts.iter().map(|s| s.as_slice()).collect(); - Some(resp_push(&refs)) - } - Command::HDel(k, fields) => match response { - Value::Integer(n) if *n > 0 => { - let mut parts: Vec<&[u8]> = vec![b"HDEL", k.as_bytes()]; - let field_refs: Vec<&[u8]> = fields.iter().map(|s| s.as_bytes()).collect(); - parts.extend_from_slice(&field_refs); - Some(resp_push(&parts)) - } - _ => None, - }, - Command::HIncrBy(k, f, _) => match response { - Value::Integer(n) => { - let s = n.to_string(); - Some(resp_push(&[ - b"HSET", - k.as_bytes(), - f.as_bytes(), - s.as_bytes(), - ])) - } - _ => None, - }, - Command::HIncrByFloat(k, f, _) => match response { - Value::BulkString(Some(data)) => { - let s = String::from_utf8_lossy(data); - Some(resp_push(&[ - b"HSET", - k.as_bytes(), - f.as_bytes(), - s.as_bytes(), - ])) - } - _ => None, - }, - Command::HSetNx(k, f, v) => match response { - Value::Integer(1) => Some(resp_push(&[ - b"HSET", - k.as_bytes(), - f.as_bytes(), - v.as_slice(), - ])), - _ => None, - }, - - // ── List ───────────────────────────────────────────────────────────── - Command::LPush(k, vals) | Command::RPush(k, vals) => { - let cmd_name = if matches!(cmd, Command::LPush(_, _)) { - "LPUSH" - } else { - "RPUSH" - }; - let mut parts: Vec<&[u8]> = vec![cmd_name.as_bytes(), k.as_bytes()]; - let val_refs: Vec<&[u8]> = vals.iter().map(|v| v.as_slice()).collect(); - parts.extend_from_slice(&val_refs); - Some(resp_push(&parts)) - } - Command::LPushX(k, vals) | Command::RPushX(k, vals) => match response { - Value::Integer(n) if *n > 0 => { - let cmd_name = if matches!(cmd, Command::LPushX(_, _)) { - "LPUSH" - } else { - "RPUSH" - }; - let mut parts: Vec<&[u8]> = vec![cmd_name.as_bytes(), k.as_bytes()]; - let val_refs: Vec<&[u8]> = vals.iter().map(|v| v.as_slice()).collect(); - parts.extend_from_slice(&val_refs); - Some(resp_push(&parts)) - } - _ => None, - }, - Command::LPop(k, count) => match response { - Value::BulkString(None) => None, - Value::Array(Some(items)) if items.is_empty() => None, - _ => { - let n = count.map(|c| c.to_string()); - match &n { - Some(ns) => Some(resp_push(&[b"LPOP", k.as_bytes(), ns.as_bytes()])), - None => Some(resp_push(&[b"LPOP", k.as_bytes()])), - } - } - }, - Command::RPop(k, count) => match response { - Value::BulkString(None) => None, - Value::Array(Some(items)) if items.is_empty() => None, - _ => { - let n = count.map(|c| c.to_string()); - match &n { - Some(ns) => Some(resp_push(&[b"RPOP", k.as_bytes(), ns.as_bytes()])), - None => Some(resp_push(&[b"RPOP", k.as_bytes()])), - } - } - }, - Command::LSet(k, idx, v) => match response { - Value::SimpleString(_) => { - let idx_s = idx.to_string(); - Some(resp_push(&[ - b"LSET", - k.as_bytes(), - idx_s.as_bytes(), - v.as_slice(), - ])) - } - _ => None, - }, - Command::LRem(k, count, elem) => match response { - Value::Integer(n) if *n > 0 => { - let count_s = count.to_string(); - Some(resp_push(&[ - b"LREM", - k.as_bytes(), - count_s.as_bytes(), - elem.as_slice(), - ])) - } - _ => None, - }, - Command::LTrim(k, start, stop) => { - let start_s = start.to_string(); - let stop_s = stop.to_string(); - Some(resp_push(&[ - b"LTRIM", - k.as_bytes(), - start_s.as_bytes(), - stop_s.as_bytes(), - ])) - } - - // ── Set ─────────────────────────────────────────────────────────────── - Command::SAdd(k, members) => match response { - Value::Integer(n) if *n > 0 => { - let mut parts: Vec<&[u8]> = vec![b"SADD", k.as_bytes()]; - let m_refs: Vec<&[u8]> = members.iter().map(|s| s.as_bytes()).collect(); - parts.extend_from_slice(&m_refs); - Some(resp_push(&parts)) - } - _ => None, - }, - Command::SRem(k, members) => match response { - Value::Integer(n) if *n > 0 => { - let mut parts: Vec<&[u8]> = vec![b"SREM", k.as_bytes()]; - let m_refs: Vec<&[u8]> = members.iter().map(|s| s.as_bytes()).collect(); - parts.extend_from_slice(&m_refs); - Some(resp_push(&parts)) - } - _ => None, - }, - Command::SPop(k, count) => { - let popped: Vec = match response { - Value::BulkString(Some(data)) => { - vec![String::from_utf8_lossy(data).into_owned()] - } - Value::Array(Some(items)) => items - .iter() - .filter_map(|v| { - if let Value::BulkString(Some(d)) = v { - Some(String::from_utf8_lossy(d).into_owned()) - } else { - None - } - }) - .collect(), - _ => vec![], - }; - if popped.is_empty() { - let _ = count; - None - } else { - let mut parts: Vec<&[u8]> = vec![b"SREM", k.as_bytes()]; - let m_refs: Vec<&[u8]> = popped.iter().map(|s| s.as_bytes()).collect(); - parts.extend_from_slice(&m_refs); - Some(resp_push(&parts)) - } - } - Command::SMove(src, dst, member) => match response { - Value::Integer(1) => Some(resp_push(&[ - b"SMOVE", - src.as_bytes(), - dst.as_bytes(), - member.as_bytes(), - ])), - _ => None, - }, - Command::SInterStore(dst, keys) => { - let mut parts: Vec<&[u8]> = vec![b"SINTERSTORE", dst.as_bytes()]; - let k_refs: Vec<&[u8]> = keys.iter().map(|s| s.as_bytes()).collect(); - parts.extend_from_slice(&k_refs); - Some(resp_push(&parts)) - } - Command::SUnionStore(dst, keys) => { - let mut parts: Vec<&[u8]> = vec![b"SUNIONSTORE", dst.as_bytes()]; - let k_refs: Vec<&[u8]> = keys.iter().map(|s| s.as_bytes()).collect(); - parts.extend_from_slice(&k_refs); - Some(resp_push(&parts)) - } - Command::SDiffStore(dst, keys) => { - let mut parts: Vec<&[u8]> = vec![b"SDIFFSTORE", dst.as_bytes()]; - let k_refs: Vec<&[u8]> = keys.iter().map(|s| s.as_bytes()).collect(); - parts.extend_from_slice(&k_refs); - Some(resp_push(&parts)) - } - - // ── Sorted Set ──────────────────────────────────────────────────────── - Command::ZAdd(k, opts, pairs) => { - let mut parts: Vec = vec!["ZADD".into(), k.clone()]; - if let Some(cond) = &opts.condition { - parts.push(match cond { - ZAddCondition::Nx => "NX".into(), - ZAddCondition::Xx => "XX".into(), - }); - } - if opts.ch { - parts.push("CH".into()); - } - if opts.incr { - parts.push("INCR".into()); - } - for (score, member) in pairs { - parts.push(format_f64_score(*score)); - parts.push(member.clone()); - } - let refs: Vec<&[u8]> = parts.iter().map(|s| s.as_bytes()).collect(); - Some(resp_push(&refs)) - } - Command::ZRem(k, members) => match response { - Value::Integer(n) if *n > 0 => { - let mut parts: Vec<&[u8]> = vec![b"ZREM", k.as_bytes()]; - let m_refs: Vec<&[u8]> = members.iter().map(|s| s.as_bytes()).collect(); - parts.extend_from_slice(&m_refs); - Some(resp_push(&parts)) - } - _ => None, - }, - Command::ZIncrBy(k, delta, member) => { - let delta_s = format_f64_score(*delta); - Some(resp_push(&[ - b"ZINCRBY", - k.as_bytes(), - delta_s.as_bytes(), - member.as_bytes(), - ])) - } - - // ── JSON ───────────────────────────────────────────────────────────── - // Replayable as-is on replicas, AOF, and browser stores. Only - // successful writes replicate (errors reply -ERR, not +OK). - Command::JSet(k, path, value) => match response { - Value::SimpleString(_) => Some(resp_push(&[ - b"JSET", - k.as_bytes(), - path.as_bytes(), - value.as_bytes(), - ])), - _ => None, - }, - Command::JMerge(k, patch) => match response { - Value::SimpleString(_) => Some(resp_push(&[b"JMERGE", k.as_bytes(), patch.as_bytes()])), - _ => None, - }, - - // ── Rate limiting ──────────────────────────────────────────────────── - // RLSET replicates so limiter *config* survives AOF replay / reaches - // replicas. RLCHECK is deliberately not replicated: attempt state is - // transient and high-frequency — streaming every check would flood the - // AOF and the sync fan-out for state that expires within one window. - Command::RlSet(k, limit, window_secs) => { - let limit_s = limit.to_string(); - let window_s = window_secs.to_string(); - Some(resp_push(&[ - b"RLSET", - k.as_bytes(), - limit_s.as_bytes(), - window_s.as_bytes(), - ])) - } - - // Pub/Sub and transactions carry no store state — no broadcast needed. - _ => None, - } -} - -fn format_f64_score(s: f64) -> String { - if s == f64::INFINITY { - "inf".into() - } else if s == f64::NEG_INFINITY { - "-inf".into() - } else if s.fract() == 0.0 && s.abs() < 1e15 { - format!("{}", s as i64) - } else { - format!("{}", s) - } -} - -/// Handles an AUTH attempt. Returns `(disconnect, resp_bytes)`. -/// -/// `disconnect` is true when the failure count hits MAX_AUTH_FAILURES. -fn process_auth( - provided: &str, - expected: &Arc>, - is_authenticated: &mut bool, - failures: &mut u32, -) -> (bool, Vec) { - match expected.as_ref() { - // Constant-time compare so a network attacker can't recover the password - // byte-by-byte from response-timing differences. - Some(pwd) if ct_eq_bytes(provided.as_bytes(), pwd.as_bytes()) => { - *is_authenticated = true; - *failures = 0; - (false, b"+OK\r\n".to_vec()) - } - Some(_) => { - *failures += 1; - if *failures >= MAX_AUTH_FAILURES { - (true, b"-ERR too many authentication failures\r\n".to_vec()) - } else { - (false, b"-ERR invalid password\r\n".to_vec()) - } - } - None => ( - false, - b"-ERR Client sent AUTH, but no password is set\r\n".to_vec(), - ), - } -} - -// ── main ───────────────────────────────────────────────────────────────────── - -#[tokio::main] -async fn main() -> Result<(), Box> { - // All runtime configuration is via RECACHED_* env vars; the only flags are - // --version/-V (required by e.g. the Homebrew formula's install test). - if std::env::args().any(|a| a == "--version" || a == "-V") { - println!("recached-server {}", env!("CARGO_PKG_VERSION")); - return Ok(()); - } - - tracing_subscriber::fmt() - .with_env_filter( - tracing_subscriber::EnvFilter::try_from_default_env() - .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")), - ) - .init(); - - // ── bind address ────────────────────────────────────────────────────── - // Host/interface all listeners bind to. Defaults to 0.0.0.0 (all - // interfaces) for backwards compatibility; set RECACHED_BIND=127.0.0.1 to - // restrict to localhost, which — together with RECACHED_PASSWORD — is - // strongly recommended unless the server is deliberately public. - let bind_host = std::env::var("RECACHED_BIND").unwrap_or_else(|_| "0.0.0.0".to_string()); - if bind_host == "0.0.0.0" { - warn!( - "Binding all interfaces (0.0.0.0). Set RECACHED_BIND=127.0.0.1 and RECACHED_PASSWORD before exposing this host." - ); - } else { - info!("Binding interface {}", bind_host); - } - - // ── Prometheus metrics ──────────────────────────────────────────────── - let metrics_port: u16 = std::env::var("RECACHED_METRICS_PORT") - .ok() - .and_then(|v| v.parse().ok()) - .unwrap_or(9091); - let metrics_addr: std::net::SocketAddr = - format!("{}:{}", bind_host, metrics_port).parse().unwrap(); - metrics_exporter_prometheus::PrometheusBuilder::new() - .with_http_listener(metrics_addr) - .install() - .expect("failed to install Prometheus metrics exporter"); - info!("Prometheus metrics at http://{}/metrics", metrics_addr); - - // ── auth ────────────────────────────────────────────────────────────── - let password = std::env::var("RECACHED_PASSWORD").ok(); - let global_password = Arc::new(password); - - if global_password.is_some() { - info!("Authentication ENABLED. Clients must send 'AUTH '."); - } else { - warn!("Authentication DISABLED. Set RECACHED_PASSWORD to enable."); - } - - // ── sync scoping ────────────────────────────────────────────────────── - let sync_secret: Arc> = Arc::new( - std::env::var("RECACHED_SYNC_SECRET") - .ok() - .filter(|s| !s.is_empty()), - ); - if sync_secret.is_some() { - info!( - "Sync scoping ENABLED (strict): WebSocket clients receive no pushes and no key access until they present 'SYNC TOKEN '." - ); - } else { - warn!( - "Sync scoping DISABLED: every WebSocket client receives every mutation. Set RECACHED_SYNC_SECRET before exposing port 6380 to untrusted clients." - ); - } - - // ── IP allowlist ────────────────────────────────────────────────────── - let allowed_ips: Option>> = match std::env::var("RECACHED_ALLOW_IPS").ok() { - None => None, - Some(raw) => match parse_allow_ips(&raw) { - Ok(ips) => Some(Arc::new(ips)), - Err(msg) => { - error!("{msg}"); - std::process::exit(1); - } - }, - }; - - if let Some(ips) = &allowed_ips { - info!("IP allowlist ENABLED: {:?}", ips); - } else { - warn!("IP allowlist DISABLED. Accepting all connections."); - } - - // ── WebSocket origin allowlist ──────────────────────────────────────── - let allowed_origins: Arc>> = - Arc::new(match std::env::var("RECACHED_ALLOWED_ORIGINS").ok() { - None => None, - Some(raw) => match parse_allowed_origins(&raw) { - Ok(list) => Some(list), - Err(msg) => { - error!("{msg}"); - std::process::exit(1); - } - }, - }); - - if let Some(list) = allowed_origins.as_ref() { - info!("WebSocket origin allowlist ENABLED: {:?}", list); - } else { - warn!( - "WebSocket origin allowlist DISABLED. Any web page a user visits can open a socket to \ - port 6380 — browsers apply neither CORS nor a preflight to WebSockets. Set \ - RECACHED_ALLOWED_ORIGINS before exposing this port to a browser." - ); - } - - // ── store ───────────────────────────────────────────────────────────── - let max_keys = std::env::var("RECACHED_MAX_KEYS") - .ok() - .and_then(|v| v.parse::().ok()); - - let max_memory_bytes = std::env::var("RECACHED_MAX_MEMORY") - .ok() - .and_then(|v| parse_memory_bytes(&v)); - - let eviction_policy = match std::env::var("RECACHED_EVICTION") - .unwrap_or_default() - .to_lowercase() - .as_str() - { - "allkeys-lru" | "lru" => EvictionPolicy::AllKeysLru, - "allkeys-random" | "random" => EvictionPolicy::AllKeysRandom, - "volatile-lru" => EvictionPolicy::VolatileLru, - "volatile-ttl" | "ttl" => EvictionPolicy::VolatileTtl, - _ => EvictionPolicy::NoEviction, - }; - - if max_keys.is_some() || max_memory_bytes.is_some() { - info!( - "Key limit: {:?}, memory limit: {:?} bytes, eviction: {:?}", - max_keys, max_memory_bytes, eviction_policy - ); - } - - let mut store_inner = KeyValueStore::with_config(max_keys, max_memory_bytes, eviction_policy); - // Eviction sample size — the knob Redis exposes as `maxmemory-samples`. - // Configured before the store is shared, so no interior mutability is needed. - store_inner.set_eviction_sample(env_limit("RECACHED_EVICTION_SAMPLE", 10)); - let store = Arc::new(store_inner); - - // ── snapshot persistence ────────────────────────────────────────────── - let save_path = PathBuf::from( - std::env::var("RECACHED_SAVE_PATH").unwrap_or_else(|_| "recached.rdb".to_string()), - ); - - // RECACHED_SAVE takes priority: "900:1,300:10,60:10000" (secs:changes pairs). - // Falls back to RECACHED_SAVE_INTERVAL (single-condition, 1 change required). - let save_conditions: Vec = if let Ok(s) = std::env::var("RECACHED_SAVE") { - parse_save_conditions(&s) - } else { - let interval: u64 = std::env::var("RECACHED_SAVE_INTERVAL") - .ok() - .and_then(|v| v.parse().ok()) - .unwrap_or(900); - if interval > 0 { - vec![SaveCondition { - secs: interval, - changes: 1, - }] - } else { - vec![] - } - }; - - load_snapshot(&store, &save_path).await; - - let snap_cfg = Arc::new(SnapshotConfig { - path: save_path, - last_save: AtomicI64::new(now_unix_secs()), - }); - - // ── AOF ─────────────────────────────────────────────────────────────── - let aof_path = std::env::var("RECACHED_AOF_PATH").ok().map(PathBuf::from); - let aof_sync = match std::env::var("RECACHED_AOF_SYNC") - .unwrap_or_default() - .to_lowercase() - .as_str() - { - "always" => AofSync::Always, - "no" => AofSync::No, - _ => AofSync::EverySec, - }; - - let aof: Option> = if let Some(path) = aof_path { - match AofWriter::open(path.clone(), aof_sync).await { - Ok(w) => { - replay_aof(&store, &path).await; - let writer = Arc::new(w); - if aof_sync == AofSync::EverySec { - let w2 = Arc::clone(&writer); - tokio::spawn(async move { - let mut interval = - tokio::time::interval(tokio::time::Duration::from_secs(1)); - loop { - interval.tick().await; - w2.flush().await; - } - }); - } - info!( - "AOF enabled: {:?} (sync={})", - path, - match aof_sync { - AofSync::Always => "always", - AofSync::EverySec => "everysec", - AofSync::No => "no", - } - ); - // `always` fsyncs inside the AOF lock, so every write in the - // process waits for one disk barrier — measured at roughly - // 20 ms per append on APFS, i.e. tens of writes per second - // rather than tens of thousands. That is the honest cost of the - // guarantee, but an operator who picked it casually will read - // the result as a hang, so say so at startup. - if aof_sync == AofSync::Always { - warn!( - "RECACHED_AOF_SYNC=always fsyncs on every write and serialises all writers \ - behind it — expect write throughput in the tens per second. Use everysec \ - unless you genuinely cannot lose one second of writes." - ); - } - Some(writer) - } - Err(e) => { - warn!("AOF open failed: {} — running without AOF", e); - None - } - } - } else { - None - }; - - // ── TLS ─────────────────────────────────────────────────────────────── - // Resolved before replication: the replication listener uses the same - // certificate, so it has to exist before that listener is spawned. - let tls_acceptor: Option = load_tls_acceptor(); - if tls_acceptor.is_some() { - info!( - "TLS ENABLED (cert={}, key={})", - std::env::var("RECACHED_TLS_CERT").unwrap_or_default(), - std::env::var("RECACHED_TLS_KEY").unwrap_or_default() - ); - } else { - warn!("TLS DISABLED. Set RECACHED_TLS_CERT and RECACHED_TLS_KEY to enable."); - } - let tls_acceptor = Arc::new(tls_acceptor); - - // ── connection limiter ──────────────────────────────────────────────── - // Resolved before replication because the replication listener shares this - // budget: a flood of replica connections must not be able to starve real - // clients, and `maxclients` should mean the total. - let max_connections = std::env::var("RECACHED_MAX_CONNECTIONS") - .ok() - .and_then(|v| v.parse::().ok()) - .unwrap_or(DEFAULT_MAX_CONNECTIONS); - info!("Max connections: {}", max_connections); - let semaphore = Arc::new(Semaphore::new(max_connections)); - - // ── Replication ─────────────────────────────────────────────────────── - let replicaof = std::env::var("RECACHED_REPLICAOF").ok(); - let repl_port: u16 = std::env::var("RECACHED_REPL_PORT") - .ok() - .and_then(|v| v.parse().ok()) - .unwrap_or(6381); - let repl_password: Option = std::env::var("RECACHED_REPL_PASSWORD").ok(); - let repl_channel_capacity: usize = std::env::var("RECACHED_REPL_BUFFER") - .ok() - .and_then(|v| v.parse().ok()) - .filter(|&n: &usize| n > 0) - .unwrap_or(DEFAULT_REPL_CHANNEL_CAPACITY); - let failover_timeout_secs: Option = std::env::var("RECACHED_FAILOVER_TIMEOUT") - .ok() - .and_then(|v| v.parse().ok()) - .filter(|&n| n > 0); - - // Whether to bind the replication listener at all. Refuses to start on a - // network-reachable interface without a password rather than serving the - // keyspace unauthenticated. - let repl_listen = match resolve_repl_listen( - std::env::var("RECACHED_REPL_ENABLE").ok(), - &bind_host, - repl_password.as_deref(), - ) { - Ok(v) => v, - Err(msg) => { - error!("{msg}"); - std::process::exit(1); - } - }; - - // Outbound replication TLS. Configured separately from the listener's TLS - // because the two directions are independent: this node may serve replicas - // over TLS, follow a primary over TLS, both, or neither. - let repl_tls: Option<(TlsConnector, String)> = match std::env::var("RECACHED_REPL_TLS_CA") - .ok() - .filter(|s| !s.is_empty()) - { - None => None, - Some(ca) => match load_repl_tls_connector(&ca) { - Ok(connector) => { - let servername = repl_tls_servername( - replicaof.as_deref().unwrap_or_default(), - std::env::var("RECACHED_REPL_TLS_SERVERNAME").ok(), - ); - info!( - "Replication client TLS ENABLED (CA={}, verifying primary as '{}')", - ca, servername - ); - Some((connector, servername)) - } - Err(msg) => { - error!("{msg}"); - std::process::exit(1); - } - }, - }; - - if replicaof.is_some() && repl_tls.is_none() { - warn!( - "Replication to the primary is PLAINTEXT — the password and the entire keyspace cross \ - the network unencrypted, and the primary's identity is not verified. Set \ - RECACHED_REPL_TLS_CA, or keep replication on a private network." - ); - } - - if !repl_listen { - info!( - "Replication server DISABLED — port {} is not bound. Set RECACHED_REPL_ENABLE=1 on any \ - node that serves replicas (including a replica serving sub-replicas).", - repl_port - ); - } else if repl_password.is_some() { - info!( - "Replication server ENABLED on port {} with auth (RECACHED_REPL_PASSWORD is set).", - repl_port - ); - } else { - warn!( - "Replication server ENABLED on port {} WITHOUT a password, on loopback only. It serves \ - the entire keyspace to whoever connects — set RECACHED_REPL_PASSWORD before binding \ - any other interface.", - repl_port - ); - } - - let is_replica_start = replicaof.is_some(); - let replicas: ReplRegistry = ReplHub::new(); - - // ── server state ────────────────────────────────────────────────────── - let state = Arc::new(ServerState { - snap: Arc::clone(&snap_cfg), - aof, - replicas: Arc::clone(&replicas), - is_replica: std::sync::atomic::AtomicBool::new(is_replica_start), - dedup: std::sync::Mutex::new(HashMap::new()), - ephemeral: std::sync::Mutex::new(HashMap::new()), - dedup_dirty: std::sync::atomic::AtomicBool::new(false), - }); - - // Restore exactly-once bookkeeping before accepting connections, so a - // client replaying an unacknowledged write after a restart is recognised - // rather than applied twice. - state.load_dedup().await; - - // ── Dedup flush ─────────────────────────────────────────────────────── - // The map is one u64 per client, so it can be persisted far more often than - // the snapshot. This bounds the duplicate window on an unclean shutdown to - // roughly this interval rather than to the snapshot cadence. - { - let state_dedup = Arc::clone(&state); - tokio::spawn(async move { - let mut ticker = tokio::time::interval(tokio::time::Duration::from_secs(1)); - ticker.tick().await; - loop { - ticker.tick().await; - state_dedup.persist_dedup().await; - } - }); - } - - // ── autosave ────────────────────────────────────────────────────────── - if !save_conditions.is_empty() { - let store_snap = Arc::clone(&store); - let state_snap = Arc::clone(&state); - let conditions = save_conditions; - tokio::spawn(async move { - let mut ticker = tokio::time::interval(tokio::time::Duration::from_secs(1)); - ticker.tick().await; // skip immediate first tick - loop { - ticker.tick().await; - let now = now_unix_secs(); - let last = state_snap.snap.last_save.load(Ordering::Relaxed); - let elapsed = now.saturating_sub(last).max(0) as u64; - let dirty = store_snap.dirty_count(); - if dirty > 0 - && conditions - .iter() - .any(|c| elapsed >= c.secs && dirty >= c.changes) - { - state_snap.save(&store_snap).await; - } - } - }); - info!("Autosave active → {:?}", snap_cfg.path); - } else { - info!( - "Autosave disabled (RECACHED_SAVE=0 or RECACHED_SAVE_INTERVAL=0). Use SAVE or BGSAVE manually." - ); - } - - // ── broadcast channel (mutation sync) ──────────────────────────────── - // Carries (sender_conn_id, resp_encoded_mutation). WS receivers skip their - // own messages. Created before replication so a replica can push the writes - // it receives from the primary to its own local WebSocket clients. - let (tx, _rx) = broadcast::channel::(BROADCAST_CHANNEL_CAPACITY); - - // ── start replication ───────────────────────────────────────────────── - // Opt-in. The listener may run on a replica as well as a primary, so a - // replica can serve sub-replicas (multi-tier replication) — but it is a - // decision the operator makes, not a port that appears by default. - if repl_listen { - let store_r = Arc::clone(&store); - let snap_r = Arc::clone(&snap_cfg); - let reg_r = Arc::clone(&replicas); - let pwd_r = repl_password.clone().map(Arc::new); - let cap_r = repl_channel_capacity; - let host_r = bind_host.clone(); - let allowed_r = allowed_ips.clone(); - let sem_r = Arc::clone(&semaphore); - let thr_r = ReplAuthThrottle::new(); - let tls_r = Arc::clone(&tls_acceptor); - tokio::spawn(async move { - run_repl_server( - host_r, repl_port, store_r, snap_r, reg_r, pwd_r, cap_r, allowed_r, sem_r, thr_r, - tls_r, - ) - .await; - }); - } - if is_replica_start && let Some(primary_addr) = replicaof { - let store_r = Arc::clone(&store); - let state_r = Arc::clone(&state); - let pwd_r = repl_password.clone(); - let fo_r = failover_timeout_secs; - let tx_r = tx.clone(); - let tls_r = repl_tls; - tokio::spawn(async move { - run_repl_client(primary_addr, store_r, state_r, pwd_r, fo_r, tx_r, tls_r).await; - }); - if let Some(t) = failover_timeout_secs { - info!( - "Running as replica — auto-failover enabled (promotes after {}s of primary being unreachable)", - t - ); - } else { - info!( - "Running as replica — write commands will be rejected (auto-failover disabled; set RECACHED_FAILOVER_TIMEOUT to enable)" - ); - } - } - - // ── background eviction ─────────────────────────────────────────────── - { - let store_sweep = Arc::clone(&store); - tokio::spawn(async move { - let mut interval = - tokio::time::interval(tokio::time::Duration::from_secs(EVICTION_INTERVAL_SECS)); - loop { - interval.tick().await; - store_sweep.sweep_expired(); - store_sweep.try_evict_for_memory(); - } - }); - } - - // ── pub/sub hub ─────────────────────────────────────────────────────── - let pubsub: SharedPubSub = Arc::new(tokio::sync::Mutex::new(PubSubHub::new())); - - // ── watch registry ──────────────────────────────────────────────────── - let watch_registry: WatchRegistry = WatchHub::new(); - - // ── Capacity & sync metrics ─────────────────────────────────────────── - // Traffic counters are event-driven, but capacity is a level, not an event: - // memory, key count and eviction rate have to be sampled. Without these an - // operator cannot answer "am I near the cap?" or "is eviction thrashing?" - // from a dashboard — see docs/server/operations.md. - { - let store_m = Arc::clone(&store); - let state_m = Arc::clone(&state); - let registry_m = watch_registry.clone(); - tokio::spawn(async move { - let mut ticker = tokio::time::interval(tokio::time::Duration::from_secs(5)); - loop { - ticker.tick().await; - // One walk of the keyspace feeds both gauges and the cached - // sample INFO reads, instead of one walk per number. - let sample = store_m.keyspace_sample(); - store_sampled_keyspace(sample); - gauge!("recached_memory_bytes").set(sample.memory_bytes as f64); - gauge!("recached_keys").set(sample.keys as f64); - counter!("recached_evictions_total").absolute(store_m.evicted_count()); - gauge!("recached_replicas_connected") - .set(state_m.replicas.count.load(Ordering::Relaxed) as f64); - gauge!("recached_replication_queue_depth") - .set(state_m.replicas.max_queue_depth().await as f64); - gauge!("recached_replication_lag_frames") - .set(state_m.replicas.max_lag_frames().await as f64); - gauge!("recached_live_queries") - .set(registry_m.watched_patterns.load(Ordering::Relaxed) as f64); - gauge!("recached_watched_keys") - .set(registry_m.watched_keys.load(Ordering::Relaxed) as f64); - gauge!("recached_dedup_clients_tracked") - .set(state_m.dedup.lock().map(|m| m.len()).unwrap_or(0) as f64); - } - }); - } - - // Startup facts for INFO. Recorded once the configuration is fully - // resolved and before any listener binds, so no connection can observe the - // defaults. - let _ = SERVER_FACTS.set(ServerFacts { - start: SystemTime::now(), - run_id: generate_run_id(), - tcp_port: 6379, - ws_port: 6380, - max_connections, - tls_enabled: tls_acceptor.is_some(), - auth_enabled: global_password.is_some(), - aof_enabled: state.aof.is_some(), - }); - - // ── listeners ───────────────────────────────────────────────────────── - let n_accept = num_cpus::get(); - let tcp_listeners = make_tcp_listeners(&format!("{}:6379", bind_host), n_accept)?; - info!( - "TCP server listening on {}:6379 ({} accept loop(s))", - bind_host, n_accept - ); - - let ws_listener = TcpListener::bind(format!("{}:6380", bind_host)).await?; - info!("WebSocket server listening on {}:6380", bind_host); - - // Spawn one accept loop per CPU core, each with its own SO_REUSEPORT socket. - // The OS load-balances incoming connections across all loops. - for tcp_listener in tcp_listeners { - let store_tcp = Arc::clone(&store); - let tx_tcp = tx.clone(); - let pass_tcp = Arc::clone(&global_password); - let allowed_tcp = allowed_ips.clone(); - let sem_tcp = Arc::clone(&semaphore); - let pubsub_tcp = Arc::clone(&pubsub); - let tls_tcp = Arc::clone(&tls_acceptor); - let watch_tcp = Arc::clone(&watch_registry); - let snap_tcp = Arc::clone(&state); - - tokio::spawn(async move { - loop { - match tcp_listener.accept().await { - Ok((socket, addr)) => { - let _ = socket.set_nodelay(true); - if let Some(allowed) = &allowed_tcp - && !allowed.contains(&addr.ip()) - { - debug!("TCP: rejected IP {}", addr.ip()); - continue; - } - let permit = match Arc::clone(&sem_tcp).try_acquire_owned() { - Ok(p) => p, - Err(_) => { - warn!("TCP: connection limit reached, dropping {}", addr); - continue; - } - }; - let s = Arc::clone(&store_tcp); - let t = tx_tcp.clone(); - let p = Arc::clone(&pass_tcp); - let ps = Arc::clone(&pubsub_tcp); - let wr = Arc::clone(&watch_tcp); - let tls = Arc::clone(&tls_tcp); - let sc = Arc::clone(&snap_tcp); - tokio::spawn(async move { - let _permit = permit; - if let Some(acc) = tls.as_ref() { - // Bounded: the permit is already held, so a peer - // that opens a socket and never negotiates would - // otherwise occupy a slot indefinitely. - match tokio::time::timeout(handshake_timeout(), acc.accept(socket)) - .await - { - Ok(Ok(tls_stream)) => { - handle_tcp( - tls_stream, - s, - t, - p, - ps, - wr, - sc, - addr.to_string(), - ) - .await - } - Ok(Err(e)) => { - warn!("TCP TLS handshake failed from {}: {}", addr, e) - } - Err(_) => { - debug!("TCP TLS handshake from {} timed out", addr) - } - } - } else { - handle_tcp(socket, s, t, p, ps, wr, sc, addr.to_string()).await; - } - }); - } - Err(e) => warn!("TCP accept error: {}", e), - } - } - }); - } - - // ── graceful shutdown via oneshot channel ──────────────────────────── - let (shutdown_tx, mut shutdown_rx) = tokio::sync::oneshot::channel::<()>(); - tokio::spawn(async move { - #[cfg(unix)] - { - let mut sigterm = - tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) - .expect("failed to register SIGTERM handler"); - tokio::select! { - _ = tokio::signal::ctrl_c() => {}, - _ = sigterm.recv() => {}, - } - } - #[cfg(not(unix))] - { - let _ = tokio::signal::ctrl_c().await; - } - let _ = shutdown_tx.send(()); - }); - - loop { - tokio::select! { - biased; - - res = ws_listener.accept() => { - match res { - Ok((socket, addr)) => { - let _ = socket.set_nodelay(true); - if let Some(allowed) = &allowed_ips - && !allowed.contains(&addr.ip()) - { - debug!("WS: rejected IP {}", addr.ip()); - continue; - } - let permit = match Arc::clone(&semaphore).try_acquire_owned() { - Ok(p) => p, - Err(_) => { - warn!("WS: connection limit reached, dropping {}", addr); - continue; - } - }; - let s = Arc::clone(&store); - let t = tx.clone(); - let p = Arc::clone(&global_password); - let ps = Arc::clone(&pubsub); - let wr = Arc::clone(&watch_registry); - let tls = Arc::clone(&tls_acceptor); - let sc = Arc::clone(&state); - let ss = Arc::clone(&sync_secret); - let ao = Arc::clone(&allowed_origins); - let id = next_conn_id(); - tokio::spawn(async move { - let _permit = permit; - if let Some(acc) = tls.as_ref() { - match tokio::time::timeout(handshake_timeout(), acc.accept(socket)).await { - Ok(Ok(tls_stream)) => { - handle_ws(tls_stream, s, t, p, id, ps, wr, sc, ss, ao, addr.to_string()).await - } - Ok(Err(e)) => warn!("WS TLS handshake failed from {}: {}", addr, e), - Err(_) => debug!("WS TLS handshake from {} timed out", addr), - } - } else { - handle_ws(socket, s, t, p, id, ps, wr, sc, ss, ao, addr.to_string()).await; - } - }); - } - Err(e) => warn!("WS accept error: {}", e), - } - } - - _ = &mut shutdown_rx => { - info!("Shutdown signal received, saving final snapshot..."); - state.save(&store).await; - info!("Done. Goodbye."); - break; - } - } - } - - Ok(()) -} - -// ── TCP handler ─────────────────────────────────────────────────────────────── - -#[allow(clippy::too_many_arguments)] -async fn handle_tcp( - socket: S, - store: Arc, - tx: broadcast::Sender, - password: Arc>, - pubsub: SharedPubSub, - watch_registry: WatchRegistry, - state: Arc, - peer: String, -) where - S: AsyncRead + AsyncWrite + Unpin + Send, -{ - let conn_id = next_conn_id(); - let mut client_meta = ClientMeta::new( - conn_id, - peer, - format!("127.0.0.1:{}", server_facts().tcp_port), - ); - let _guard = ConnectionGuard::new("tcp", client_meta.clone()); - let (mut reader, raw_writer) = tokio::io::split(socket); - let mut writer = tokio::io::BufWriter::with_capacity(32 * 1024, raw_writer); - let mut buf = Vec::::new(); - let mut read_pos: usize = 0; - // Bytes the in-flight frame needs before another parse attempt can - // possibly succeed; 0 means "try now". See the gate in the read arm. - let mut need: usize = 0; - let mut read_buf = [0u8; TCP_READ_BUFFER_BYTES]; - // Reused for every response on this connection — avoids a Vec allocation - // per command (significant under pipelining). - let mut resp_buf = Vec::::with_capacity(4 * 1024); - let mut is_authenticated = password.is_none(); - let mut auth_failures: u32 = 0; - // RESP2 until the client negotiates otherwise. Defaulting to 2 keeps every - // existing client working: they never send HELLO and must not start - // receiving RESP3-only types. - let mut protover: u8 = 2; - let mut multi_queue: Option> = None; - let mut subscribed_channels: HashSet = HashSet::new(); - let mut subscribed_patterns: HashSet = HashSet::new(); - let (ps_tx, mut ps_rx) = mpsc::unbounded_channel::(); - // WATCH state for optimistic-lock transactions over TCP. Unlike the WS - // handler, TCP clients are not sent keychange pushes — WATCH is pure CAS. - let mut watched_keys: HashSet = HashSet::new(); - let mut watch_dirty = false; - let (watch_tx, mut watch_rx) = mpsc::unbounded_channel::(); - - 'outer: loop { - let is_subscribed = !subscribed_channels.is_empty() || !subscribed_patterns.is_empty(); - // Republish only when the counts moved: CLIENT LIST has to see live - // subscription state, but taking the registry's write lock once per - // command would put it on the hot path. - if client_meta.sub != subscribed_channels.len() - || client_meta.psub != subscribed_patterns.len() - { - client_meta.sub = subscribed_channels.len(); - client_meta.psub = subscribed_patterns.len(); - publish_client(client_meta.clone()); - } - - tokio::select! { - result = reader.read(&mut read_buf) => { - match result { - Ok(0) => break, - Ok(n) => { - if (buf.len() - read_pos) + n > MAX_TCP_READ_BUFFER_BYTES { - warn!("TCP connection exceeded max buffer size, closing"); - break 'outer; - } - buf.extend_from_slice(&read_buf[..n]); - // A frame that cannot possibly be complete is not worth - // re-parsing. `Value::parse` starts from the beginning - // every time, rebuilding — and reallocating — every - // bulk string it has already seen, so a large multi-bulk - // arriving over hundreds of segments used to re-copy - // everything received so far on each one. `need` is the - // parser's lower bound on the finished frame; until the - // buffer holds that much, skip the work entirely. - if buf.len() - read_pos < need { - continue 'outer; - } - 'parse: loop { - // Completeness is decided by the non-allocating - // measure. `Value::parse` restarts from the first - // byte every call, so asking *it* whether a frame - // had arrived meant rebuilding — and reallocating — - // every element received so far, once per segment, - // and discarding all of it. `frame_len` walks the - // headers and steps over payloads; `parse` below - // then runs once, on a frame known to be whole. - match Value::frame_len(&buf[read_pos..]) { - Ok(_) => {} - Err(e) if e.is_incomplete() => { - // Measured from `read_pos`, and compaction - // moves that to 0, so the bound stays valid. - need = e.needed(); - // Compact: drop already-parsed bytes. - buf.drain(..read_pos); - read_pos = 0; - break 'parse; - } - Err(e) => { - warn!("TCP protocol error: {}", e); - let _ = writer.write_all(b"-ERR Protocol error\r\n").await; - buf.clear(); - read_pos = 0; - need = 0; - break 'parse; - } - } - match Value::parse(&buf[read_pos..]) { - Ok((value, consumed)) => { - read_pos += consumed; - let cmd = match Command::from_value(value) { - Ok(c) => c, - Err(e) => { - let r = Value::Error(e).serialize(); - if writer.write_all(&r).await.is_err() { break 'outer; } - continue 'parse; - } - }; - - // AUTH is always processed immediately - if let Command::Auth(ref pwd) = cmd { - let (disconnect, resp) = process_auth( - pwd, &password, &mut is_authenticated, &mut auth_failures, - ); - if writer.write_all(&resp).await.is_err() { break 'outer; } - if disconnect { - let _ = writer.flush().await; - break 'outer; - } - continue 'parse; - } - - if let Command::Hello(ref requested) = cmd { - let resp = process_hello( - requested.as_deref(), - &mut protover, - is_authenticated, - state.is_replica(), - ); - if writer.write_all(&resp).await.is_err() { break 'outer; } - // CLIENT LIST reports resp= per connection, - // so a renegotiation has to reach the registry. - if client_meta.resp != protover { - client_meta.resp = protover; - publish_client(client_meta.clone()); - } - continue 'parse; - } - - // QUIT is answered before the auth gate and - // before the subscribe-mode gate, as in Redis: - // a client that cannot authenticate, or is - // parked in subscribe mode, still deserves a - // clean close rather than a dropped socket. - if matches!(cmd, Command::Quit) { - let _ = writer.write_all(b"+OK\r\n").await; - let _ = writer.flush().await; - break 'outer; - } - - if !is_authenticated { - if writer.write_all(b"-NOAUTH Authentication required.\r\n").await.is_err() { - break 'outer; - } - continue 'parse; - } - - // ── Transactions ────────────────────────────── - match &cmd { - Command::Multi => { - let resp = if multi_queue.is_some() { - b"-ERR MULTI calls can not be nested\r\n".to_vec() - } else { - multi_queue = Some(Vec::new()); - b"+OK\r\n".to_vec() - }; - if writer.write_all(&resp).await.is_err() { break 'outer; } - continue 'parse; - } - Command::Discard => { - let resp = if multi_queue.take().is_some() { - // DISCARD also flushes WATCH state. - unregister_all_watches(&watch_registry, conn_id, &mut watched_keys).await; - while watch_rx.try_recv().is_ok() {} - watch_dirty = false; - b"+OK\r\n".to_vec() - } else { - b"-ERR DISCARD without MULTI\r\n".to_vec() - }; - if writer.write_all(&resp).await.is_err() { break 'outer; } - continue 'parse; - } - Command::Exec => { - match multi_queue.take() { - None => { - if writer.write_all(b"-ERR EXEC without MULTI\r\n").await.is_err() { break 'outer; } - } - Some(queue) => { - // Drain pending notifications so the CAS check isn't racy. - while watch_rx.try_recv().is_ok() { - watch_dirty = true; - } - if watch_dirty { - // A watched key changed since WATCH — abort with nil array. - unregister_all_watches(&watch_registry, conn_id, &mut watched_keys).await; - while watch_rx.try_recv().is_ok() {} - watch_dirty = false; - if writer.write_all(&Value::Array(None).serialize()).await.is_err() { break 'outer; } - } else { - let mut results = Vec::with_capacity(queue.len()); - let armed = write_effects_armed(&tx, &state, &watch_registry); - for qcmd in queue { - let resp = if armed && is_write_command(&qcmd) { - let resp = execute_and_record(&store, qcmd.clone()); - apply_write_effects(&qcmd, &resp, &tx, 0, &state, &watch_registry, &store).await; - resp - } else { - execute_and_record(&store, qcmd) - }; - results.push(resp); - } - unregister_all_watches(&watch_registry, conn_id, &mut watched_keys).await; - while watch_rx.try_recv().is_ok() {} - watch_dirty = false; - let out = Value::Array(Some(results)).serialize(); - if writer.write_all(&out).await.is_err() { break 'outer; } - } - } - } - continue 'parse; - } - _ => {} - } - - // If inside MULTI, queue the command - if let Some(ref mut queue) = multi_queue { - // Pub/sub and WATCH commands cannot be queued - match &cmd { - Command::Subscribe(_) | Command::Unsubscribe(_) - | Command::PSubscribe(_) | Command::PUnsubscribe(_) - | Command::Publish(_, _) - | Command::Watch(_) | Command::Unwatch(_) - | Command::QSub(_) | Command::QUnsub(_) => { - let err = b"-ERR Command not allowed inside a transaction\r\n"; - if writer.write_all(err).await.is_err() { break 'outer; } - } - _ => { - if queue.len() >= max_multi_queue_len() { - let err = b"-ERR transaction queue limit reached\r\n"; - if writer.write_all(err).await.is_err() { break 'outer; } - } else { - queue.push(cmd); - if writer.write_all(b"+QUEUED\r\n").await.is_err() { break 'outer; } - } - } - } - continue 'parse; - } - - // ── Pub/Sub commands ────────────────────────── - match cmd { - Command::Subscribe(channels) => { - for ch in channels { - subscribed_channels.insert(ch.clone()); - pubsub.lock().await.subscribe(conn_id, &ch, ps_tx.clone()); - let count = subscribed_channels.len() + subscribed_patterns.len(); - let ack = resp_subscribe_ack("subscribe", &ch, count); - if writer.write_all(&ack).await.is_err() { break 'outer; } - } - } - Command::Unsubscribe(channels) => { - let targets: Vec = if channels.is_empty() { - subscribed_channels.drain().collect() - } else { - channels.into_iter().filter(|c| subscribed_channels.remove(c)).collect() - }; - for ch in &targets { - pubsub.lock().await.unsubscribe(conn_id, ch); - let count = subscribed_channels.len() + subscribed_patterns.len(); - let ack = resp_subscribe_ack("unsubscribe", ch, count); - if writer.write_all(&ack).await.is_err() { break 'outer; } - } - if targets.is_empty() { - let ack = resp_subscribe_ack("unsubscribe", "", 0); - if writer.write_all(&ack).await.is_err() { break 'outer; } - } - } - Command::PSubscribe(patterns) => { - for pat in patterns { - subscribed_patterns.insert(pat.clone()); - pubsub.lock().await.psubscribe(conn_id, &pat, ps_tx.clone()); - let count = subscribed_channels.len() + subscribed_patterns.len(); - let ack = resp_subscribe_ack("psubscribe", &pat, count); - if writer.write_all(&ack).await.is_err() { break 'outer; } - } - } - Command::PUnsubscribe(patterns) => { - let targets: Vec = if patterns.is_empty() { - subscribed_patterns.drain().collect() - } else { - patterns.into_iter().filter(|p| subscribed_patterns.remove(p)).collect() - }; - for pat in &targets { - pubsub.lock().await.punsubscribe(conn_id, pat); - let count = subscribed_channels.len() + subscribed_patterns.len(); - let ack = resp_subscribe_ack("punsubscribe", pat, count); - if writer.write_all(&ack).await.is_err() { break 'outer; } - } - if targets.is_empty() { - let ack = resp_subscribe_ack("punsubscribe", "", 0); - if writer.write_all(&ack).await.is_err() { break 'outer; } - } - } - Command::Publish(channel, message) => { - let count = pubsub.lock().await.publish(&channel, &message); - let resp = Value::Integer(count).serialize(); - if writer.write_all(&resp).await.is_err() { break 'outer; } - } - - Command::Watch(keys) => { - let new_count = keys.iter().filter(|k| !watched_keys.contains(*k)).count(); - if watched_keys.len() + new_count > max_watches_per_conn() { - if writer.write_all(b"-ERR watch limit per connection reached\r\n").await.is_err() { break 'outer; } - } else { - { - let mut reg = watch_registry.map.lock().await; - for key in &keys { - if watched_keys.insert(key.clone()) { - reg.entry(key.clone()).or_default().push((conn_id, watch_tx.clone())); - } - } - watch_registry.sync_len(®); - } - if writer.write_all(b"+OK\r\n").await.is_err() { break 'outer; } - } - } - Command::Unwatch(keys) => { - let targets: Vec = if keys.is_empty() { - watched_keys.drain().collect() - } else { - keys.into_iter().filter(|k| watched_keys.remove(k)).collect() - }; - { - let mut reg = watch_registry.map.lock().await; - for key in &targets { - if let Some(subs) = reg.get_mut(key) { - subs.retain(|(id, _)| *id != conn_id); - if subs.is_empty() { reg.remove(key); } - } - } - watch_registry.sync_len(®); - } - if watched_keys.is_empty() { - while watch_rx.try_recv().is_ok() {} - watch_dirty = false; - } - if writer.write_all(b"+OK\r\n").await.is_err() { break 'outer; } - } - - cmd => { - // In subscribe mode only ping is allowed - if is_subscribed && !matches!(cmd, Command::Ping(_)) { - let err = b"-ERR only (P)SUBSCRIBE / (P)UNSUBSCRIBE / PING / QUIT allowed in subscribe mode\r\n"; - if writer.write_all(err).await.is_err() { break 'outer; } - continue 'parse; - } - // Replica: reject writes - if state.is_replica() && is_write_command(&cmd) { - let err = b"-READONLY You can't write against a read only replica.\r\n"; - if writer.write_all(err).await.is_err() { break 'outer; } - continue 'parse; - } - // Snapshot commands — handled here (async I/O, not in execute()) - match &cmd { - Command::Save => { - state.save(&store).await; - if writer.write_all(b"+OK\r\n").await.is_err() { break 'outer; } - continue 'parse; - } - Command::BgSave => { - let s = Arc::clone(&store); - let st = Arc::clone(&state); - tokio::spawn(async move { st.save(&s).await; }); - if writer.write_all(b"+Background saving started\r\n").await.is_err() { break 'outer; } - continue 'parse; - } - Command::LastSave => { - let ts = state.snap.last_save.load(Ordering::Relaxed); - if writer.write_all(&Value::Integer(ts).serialize()).await.is_err() { break 'outer; } - continue 'parse; - } - Command::Info(sections) => { - let repl = ReplInfo { - connected: state.replicas.count.load(Ordering::Relaxed), - queue_depth: state.replicas.max_queue_depth().await, - lag_frames: state.replicas.max_lag_frames().await, - }; - let body = render_info( - sections, - server_facts(), - &store, - sampled_keyspace(&store), - state.is_replica(), - repl, - state.snap.last_save.load(Ordering::Relaxed), - watch_registry.watched_patterns.load(Ordering::Relaxed) as u64, - watch_registry.watched_keys.load(Ordering::Relaxed) as u64, - ); - let resp = Value::BulkString(Some(body.into_bytes())).serialize(); - if writer.write_all(&resp).await.is_err() { break 'outer; } - continue 'parse; - } - Command::Client(args) => { - let resp = handle_client_command(args, &mut client_meta).serialize(); - if writer.write_all(&resp).await.is_err() { break 'outer; } - continue 'parse; - } - Command::Config(args) => { - let resp = handle_config_command(args, server_facts(), &store).serialize(); - if writer.write_all(&resp).await.is_err() { break 'outer; } - continue 'parse; - } - Command::CommandQuery(args) => { - let resp = handle_command_query(args, protover).serialize(); - if writer.write_all(&resp).await.is_err() { break 'outer; } - continue 'parse; - } - Command::Cluster(args) => { - let resp = handle_cluster_command(args).serialize(); - if writer.write_all(&resp).await.is_err() { break 'outer; } - continue 'parse; - } - Command::Module(args) => { - let resp = handle_module_command(args).serialize(); - if writer.write_all(&resp).await.is_err() { break 'outer; } - continue 'parse; - } - Command::Memory(args) => { - let resp = handle_memory_command(args).serialize(); - if writer.write_all(&resp).await.is_err() { break 'outer; } - continue 'parse; - } - Command::PubSub(args) => { - let resp = handle_pubsub_command(args, &*pubsub.lock().await).serialize(); - if writer.write_all(&resp).await.is_err() { break 'outer; } - continue 'parse; - } - Command::ReplicaOfNoOne => { - state.promote_to_primary(); - if writer.write_all(b"+OK\r\n").await.is_err() { break 'outer; } - continue 'parse; - } - _ => {} - } - let response = if is_write_command(&cmd) - && write_effects_armed(&tx, &state, &watch_registry) - { - let response = execute_and_record(&store, cmd.clone()); - apply_write_effects(&cmd, &response, &tx, 0, &state, &watch_registry, &store).await; - response - } else { - execute_and_record(&store, cmd) - }; - resp_buf.clear(); - response.serialize_into(&mut resp_buf); - if writer.write_all(&resp_buf).await.is_err() { - break 'outer; - } - } - } - } - Err(e) if e.is_incomplete() => { - // Measured from `read_pos`, and compaction - // moves that to 0, so the bound stays valid. - need = e.needed(); - // Compact: drop already-parsed bytes, reset cursor. - buf.drain(..read_pos); - read_pos = 0; - break 'parse; - } - Err(e) => { - warn!("TCP protocol error: {}", e); - let _ = writer.write_all(b"-ERR Protocol error\r\n").await; - buf.clear(); - read_pos = 0; - need = 0; - break 'parse; - } - } - } - // Flush all responses for this read batch in one syscall. - if writer.flush().await.is_err() { - break 'outer; - } - } - Err(e) => { - warn!("TCP read error: {}", e); - break; - } - } - } - - msg = ps_rx.recv(), if is_subscribed => { - match msg { - Some(m) => { - if writer.write_all(&encode_pubsub_msg(m, protover)).await.is_err() { - break; - } - // `writer` is a BufWriter, and a delivery is not a - // response to anything this connection sent — nothing - // else is going to flush it. Without this a subscriber - // that only listens receives nothing until it happens - // to send a command or 32 KB of pushes accumulate. - if writer.flush().await.is_err() { - break; - } - } - None => break, - } - } - - // A watched key changed: mark the transaction dirty so a following - // EXEC aborts. TCP clients get no keychange push (WATCH is pure CAS). - notif = watch_rx.recv(), if !watched_keys.is_empty() => { - if notif.is_some() { - watch_dirty = true; - } - } - } - } - - if !subscribed_channels.is_empty() || !subscribed_patterns.is_empty() { - pubsub.lock().await.unsubscribe_all(conn_id); - } - unregister_all_watches(&watch_registry, conn_id, &mut watched_keys).await; -} - -// ── WebSocket handler ───────────────────────────────────────────────────────── - -/// Complete the WebSocket handshake, enforcing the origin allowlist and a -/// deadline. Returns `None` when the connection was refused, failed, or stalled -/// — in every case the caller simply drops the socket and its permit. -/// -/// Split out from `handle_ws` so both the origin decision and the timeout are -/// reachable from a test without standing up a listener. -/// -/// `result_large_err`: the error type is tungstenite's `ErrorResponse`, which is -/// an `http::Response` — its size is the handshake callback's signature, not -/// ours, and boxing it would not satisfy the trait. -#[allow(clippy::result_large_err)] -async fn ws_handshake( - socket: S, - allowed_origins: Option<&[String]>, - timeout: Duration, - conn_id: u64, -) -> Option> -where - S: AsyncRead + AsyncWrite + Unpin, -{ - let check_origin = |req: &HandshakeRequest, - resp: HandshakeResponse| - -> Result { - let origin = req - .headers() - .get("origin") - .and_then(|v| v.to_str().ok()) - .map(str::to_owned); - if origin_allowed(allowed_origins, origin.as_deref()) { - return Ok(resp); - } - warn!( - "WS conn {}: refused Origin {:?} — not in RECACHED_ALLOWED_ORIGINS", - conn_id, origin - ); - let mut err = ErrorResponse::new(Some( - "Origin not allowed. Add it to RECACHED_ALLOWED_ORIGINS to permit this page." - .to_string(), - )); - *err.status_mut() = StatusCode::FORBIDDEN; - Err(err) - }; - - match tokio::time::timeout(timeout, accept_hdr_async(socket, check_origin)).await { - Ok(Ok(ws)) => Some(ws), - Ok(Err(e)) => { - warn!("WS handshake failed on conn {}: {}", conn_id, e); - None - } - Err(_) => { - debug!("WS handshake on conn {} timed out", conn_id); - None - } - } -} - -#[allow(clippy::too_many_arguments)] -async fn handle_ws( - socket: S, - store: Arc, - tx: broadcast::Sender, - password: Arc>, - conn_id: u64, - pubsub: SharedPubSub, - watch_registry: WatchRegistry, - state: Arc, - sync_secret: Arc>, - allowed_origins: Arc>>, - peer: String, -) where - S: AsyncRead + AsyncWrite + Unpin + Send, -{ - let mut client_meta = ClientMeta::new( - conn_id, - peer, - format!("127.0.0.1:{}", server_facts().ws_port), - ); - // The WebSocket transport speaks RESP3 from the first frame. - client_meta.resp = 3; - let _guard = ConnectionGuard::new("ws", client_meta.clone()); - let Some(ws_stream) = ws_handshake( - socket, - allowed_origins.as_deref(), - handshake_timeout(), - conn_id, - ) - .await - else { - return; - }; - - let (mut ws_sender, mut ws_receiver) = ws_stream.split(); - let mut rx = tx.subscribe(); - let mut is_authenticated = password.is_none(); - let mut auth_failures: u32 = 0; - let mut multi_queue: Option> = None; - let mut subscribed_channels: HashSet = HashSet::new(); - let mut subscribed_patterns: HashSet = HashSet::new(); - let (ps_tx, mut ps_rx) = mpsc::unbounded_channel::(); - let mut watched_keys: HashSet = HashSet::new(); - // Set when any watched key changes; EXEC aborts (returns nil) if true. - let mut watch_dirty = false; - let (watch_tx, mut watch_rx) = mpsc::unbounded_channel::(); - // Sync scopes for this connection. `strict` (RECACHED_SYNC_SECRET set) - // means: no pushes and no key commands until a signed token is presented. - // Without a secret, scopes are an opt-in bandwidth filter (legacy fan-out - // of everything when None). - let strict = sync_secret.is_some(); - let mut sync_scopes: Option> = None; - // Live-query subscriptions (QSUB). Keychange notifications for matching - // keys arrive on their own channel so they never dirty WATCH transactions. - let mut qsub_patterns: HashSet = HashSet::new(); - let (q_tx, mut q_rx) = mpsc::unbounded_channel::(); - - // Replies go out as *text* frames whenever the RESP bytes are valid UTF-8, - // which is the overwhelming majority and is what every existing client - // expects. A reply carrying a value that is not valid UTF-8 goes out as a - // *binary* frame instead of being mangled by a lossy conversion, which is - // what made raw binary values round-trip only over the TCP port. - macro_rules! ws_send { - ($bytes:expr) => {{ - let bytes: &[u8] = $bytes; - let msg = match std::str::from_utf8(bytes) { - Ok(text) => Message::Text(text.into()), - Err(_) => Message::Binary(bytes.to_vec().into()), - }; - if ws_sender.send(msg).await.is_err() { - break; - } - }}; - } - - 'outer: loop { - let is_subscribed = !subscribed_channels.is_empty() || !subscribed_patterns.is_empty(); - - tokio::select! { - msg = ws_receiver.next() => { - match msg { - // Binary frames carry the same RESP bytes as text frames. - // They exist so a client can write a value that is not - // valid UTF-8 — impossible over a text frame, which the - // WebSocket spec requires to be well-formed UTF-8. - Some(Ok(frame @ (Message::Text(_) | Message::Binary(_)))) => { - let raw: Vec = match &frame { - Message::Text(t) => t.as_bytes().to_vec(), - Message::Binary(b) => b.to_vec(), - _ => unreachable!("pattern restricts to text and binary"), - }; - let (value, _) = match Value::parse(&raw) { - Ok(v) => v, - Err(e) => { - let err = Value::Error(format!("ERR Protocol error: {}", e)).serialize(); - ws_send!(&err); - continue; - } - }; - - let cmd = match Command::from_value(value) { - Ok(c) => c, - Err(e) => { - let err = Value::Error(e).serialize(); - ws_send!(&err); - continue; - } - }; - - // AUTH - if let Command::Auth(ref pwd) = cmd { - let (disconnect, resp) = process_auth( - pwd, &password, &mut is_authenticated, &mut auth_failures, - ); - ws_send!(&resp); - if disconnect { break; } - continue; - } - - // The WebSocket sync protocol is specified in terms of - // RESP3 push frames, so this transport is always RESP3 - // and HELLO cannot downgrade it — a client asking for 2 - // is refused rather than silently left on 3. - if let Command::Hello(ref requested) = cmd { - let mut ws_protover: u8 = 3; - let resp = match requested.as_deref() { - Some("2") => Value::Error( - "NOPROTO the WebSocket transport requires RESP3".to_string(), - ) - .serialize(), - other => process_hello( - other, - &mut ws_protover, - is_authenticated, - state.is_replica(), - ), - }; - ws_send!(&resp); - continue; - } - - if matches!(cmd, Command::Quit) { - ws_send!(b"+OK\r\n"); - break 'outer; - } - - if !is_authenticated { - let resp = Value::Error("NOAUTH Authentication required.".to_string()).serialize(); - ws_send!(&resp); - continue; - } - - // ── Sync scoping ────────────────────────────────────── - if let Command::Sync(ref args) = cmd { - let resp = handle_sync_command(args, (*sync_secret).as_deref(), &mut sync_scopes, conn_id); - ws_send!(&resp); - continue; - } - // Token-scoped mode: check every command against this - // connection's granted scopes before it runs (including - // commands about to be queued inside MULTI). - if strict { - match command_scope(&cmd) { - CommandScope::KeyLess => {} - CommandScope::Admin => { - ws_send!(b"-NOSCOPE keyspace-wide and administrative commands are not available on scoped WebSocket connections\r\n"); - continue; - } - CommandScope::Keys(keys) => { - let Some(ref scopes) = sync_scopes else { - ws_send!(b"-NOSCOPE send SYNC TOKEN before issuing commands\r\n"); - continue; - }; - if let Some(denied) = keys - .iter() - .find(|k| !scopes_match(scopes, std::slice::from_ref(k))) - { - let err = Value::Error(format!( - "NOSCOPE key '{}' is outside this connection's sync scopes", - denied - )) - .serialize(); - ws_send!(&err); - continue; - } - } - } - } - - // ── Transactions ────────────────────────────────────── - match &cmd { - Command::Multi => { - let resp = if multi_queue.is_some() { - b"-ERR MULTI calls can not be nested\r\n".to_vec() - } else { - multi_queue = Some(Vec::new()); - b"+OK\r\n".to_vec() - }; - ws_send!(&resp); - continue; - } - Command::Discard => { - let resp = if multi_queue.take().is_some() { - // DISCARD also flushes WATCH state. - unregister_all_watches(&watch_registry, conn_id, &mut watched_keys).await; - while watch_rx.try_recv().is_ok() {} // drop stale notifications - watch_dirty = false; - b"+OK\r\n".to_vec() - } else { - b"-ERR DISCARD without MULTI\r\n".to_vec() - }; - ws_send!(&resp); - continue; - } - Command::Exec => { - match multi_queue.take() { - None => { - ws_send!(b"-ERR EXEC without MULTI\r\n"); - } - Some(queue) => { - // Catch watched-key changes that arrived but the select - // loop hasn't drained yet, so the CAS check isn't racy. - while watch_rx.try_recv().is_ok() { - watch_dirty = true; - } - if watch_dirty { - // A watched key changed since WATCH — abort: return - // a nil array and run nothing (Redis CAS semantics). - unregister_all_watches(&watch_registry, conn_id, &mut watched_keys).await; - while watch_rx.try_recv().is_ok() {} // drop stale notifications - watch_dirty = false; - ws_send!(&Value::Array(None).serialize()); - } else { - let mut results = Vec::with_capacity(queue.len()); - let armed = write_effects_armed(&tx, &state, &watch_registry); - for qcmd in queue { - let resp = if armed && is_write_command(&qcmd) { - let resp = execute_and_record(&store, qcmd.clone()); - apply_write_effects(&qcmd, &resp, &tx, conn_id, &state, &watch_registry, &store).await; - resp - } else { - execute_and_record(&store, qcmd) - }; - results.push(resp); - } - // EXEC always flushes WATCH state. Drain any - // self-notifications the queued writes produced so - // they can't dirty a later transaction. - unregister_all_watches(&watch_registry, conn_id, &mut watched_keys).await; - while watch_rx.try_recv().is_ok() {} - watch_dirty = false; - let out = Value::Array(Some(results)).serialize(); - ws_send!(&out); - } - } - } - continue; - } - _ => {} - } - - // Queue if inside MULTI - if let Some(ref mut queue) = multi_queue { - match &cmd { - Command::Subscribe(_) | Command::Unsubscribe(_) - | Command::PSubscribe(_) | Command::PUnsubscribe(_) - | Command::Publish(_, _) - | Command::Watch(_) | Command::Unwatch(_) - | Command::QSub(_) | Command::QUnsub(_) => { - ws_send!(b"-ERR Command not allowed inside a transaction\r\n"); - } - _ => { - if queue.len() >= max_multi_queue_len() { - ws_send!(b"-ERR transaction queue limit reached\r\n"); - } else { - queue.push(cmd); - ws_send!(b"+QUEUED\r\n"); - } - } - } - continue; - } - - // ── Pub/Sub commands ────────────────────────────────── - match cmd { - Command::Subscribe(channels) => { - for ch in channels { - subscribed_channels.insert(ch.clone()); - pubsub.lock().await.subscribe(conn_id, &ch, ps_tx.clone()); - let count = subscribed_channels.len() + subscribed_patterns.len(); - ws_send!(&resp_subscribe_ack("subscribe", &ch, count)); - } - } - Command::Unsubscribe(channels) => { - let targets: Vec = if channels.is_empty() { - subscribed_channels.drain().collect() - } else { - channels.into_iter().filter(|c| subscribed_channels.remove(c)).collect() - }; - for ch in &targets { - pubsub.lock().await.unsubscribe(conn_id, ch); - let count = subscribed_channels.len() + subscribed_patterns.len(); - ws_send!(&resp_subscribe_ack("unsubscribe", ch, count)); - } - if targets.is_empty() { - ws_send!(&resp_subscribe_ack("unsubscribe", "", 0)); - } - } - Command::PSubscribe(patterns) => { - for pat in patterns { - subscribed_patterns.insert(pat.clone()); - pubsub.lock().await.psubscribe(conn_id, &pat, ps_tx.clone()); - let count = subscribed_channels.len() + subscribed_patterns.len(); - ws_send!(&resp_subscribe_ack("psubscribe", &pat, count)); - } - } - Command::PUnsubscribe(patterns) => { - let targets: Vec = if patterns.is_empty() { - subscribed_patterns.drain().collect() - } else { - patterns.into_iter().filter(|p| subscribed_patterns.remove(p)).collect() - }; - for pat in &targets { - pubsub.lock().await.punsubscribe(conn_id, pat); - let count = subscribed_channels.len() + subscribed_patterns.len(); - ws_send!(&resp_subscribe_ack("punsubscribe", pat, count)); - } - if targets.is_empty() { - ws_send!(&resp_subscribe_ack("punsubscribe", "", 0)); - } - } - Command::Publish(channel, message) => { - let count = pubsub.lock().await.publish(&channel, &message); - ws_send!(&Value::Integer(count).serialize()); - } - - Command::Watch(keys) => { - let new_count = keys - .iter() - .filter(|k| !watched_keys.contains(*k)) - .count(); - if watched_keys.len() + new_count > max_watches_per_conn() { - ws_send!(b"-ERR watch limit per connection reached\r\n"); - } else { - { - let mut reg = watch_registry.map.lock().await; - for key in &keys { - if watched_keys.insert(key.clone()) { - reg.entry(key.clone()) - .or_default() - .push((conn_id, watch_tx.clone())); - } - } - watch_registry.sync_len(®); - } // reg dropped before await - ws_send!(b"+OK\r\n"); - } - } - Command::Unwatch(keys) => { - let targets: Vec = if keys.is_empty() { - watched_keys.drain().collect() - } else { - keys.into_iter().filter(|k| watched_keys.remove(k)).collect() - }; - { - let mut reg = watch_registry.map.lock().await; - for key in &targets { - if let Some(subs) = reg.get_mut(key) { - subs.retain(|(id, _)| *id != conn_id); - if subs.is_empty() { - reg.remove(key); - } - } - } - watch_registry.sync_len(®); - } - // Once nothing is watched, clear the dirty flag and drop any - // queued notifications so a later WATCH/MULTI/EXEC starts clean. - if watched_keys.is_empty() { - while watch_rx.try_recv().is_ok() {} - watch_dirty = false; - } - ws_send!(b"+OK\r\n"); - } - - Command::QSub(pattern) => { - // Strict mode: the requested pattern must sit inside a - // granted scope. A grant covers the request when it is - // identical or glob-matches the request as literal text - // (prefix-style grants: `cart:*` covers `cart:42:*`). - if strict { - let allowed = sync_scopes.as_ref().is_some_and(|scopes| { - scopes.iter().any(|s| { - s == &pattern - || core_engine::store::glob_match(s, &pattern) - }) - }); - if !allowed { - ws_send!(b"-NOSCOPE pattern is outside this connection's sync scopes\r\n"); - continue 'outer; - } - } - if !qsub_patterns.contains(&pattern) - && qsub_patterns.len() >= max_qsubs_per_conn() - { - ws_send!(b"-ERR live query limit per connection reached\r\n"); - continue 'outer; - } - // Register *before* snapshotting: a write landing in - // between is delivered as a keychange after the initial - // state, which is idempotent — the reverse order would - // lose it. - if qsub_patterns.insert(pattern.clone()) { - let mut pats = watch_registry.patterns.lock().await; - pats.entry(pattern.clone()) - .or_default() - .push((conn_id, q_tx.clone())); - watch_registry.sync_patterns_len(&pats); - } - let kvs = store.matching_key_values(&pattern, max_qsub_initial_keys()); - // Tagged reply so clients can recognise it among - // interleaved frames: ["qstate", pattern, k, v, ...] - let mut items = Vec::with_capacity(kvs.len() * 2 + 2); - items.push(Value::BulkString(Some(b"qstate".to_vec()))); - items.push(Value::BulkString(Some(pattern.clone().into_bytes()))); - for (k, v) in kvs { - items.push(Value::BulkString(Some(k.into_bytes()))); - items.push(v); - } - ws_send!(&Value::Array(Some(items)).serialize()); - } - Command::QUnsub(pattern) => { - let targets: Vec = match pattern { - Some(p) => { - if qsub_patterns.remove(&p) { - vec![p] - } else { - vec![] - } - } - None => qsub_patterns.drain().collect(), - }; - if !targets.is_empty() { - let mut pats = watch_registry.patterns.lock().await; - for p in &targets { - if let Some(subs) = pats.get_mut(p) { - subs.retain(|(id, _)| *id != conn_id); - if subs.is_empty() { - pats.remove(p); - } - } - } - watch_registry.sync_patterns_len(&pats); - } - ws_send!(b"+OK\r\n"); - } - - cmd => { - // Exactly-once: unwrap the DEDUP envelope. An id at or - // below this client's high-water mark was already applied - // (its acknowledgment was lost) — skip it. +DUP still - // acknowledges the write so the client retires it. - let cmd = match cmd { - Command::Dedup(client, id, inner) => { - if state.dedup_seen(&client, id) { - ws_send!(b"+DUP\r\n"); - continue 'outer; - } - *inner - } - other => other, - }; - if is_subscribed && !matches!(cmd, Command::Ping(_)) { - ws_send!(b"-ERR only (P)SUBSCRIBE / (P)UNSUBSCRIBE / PING / QUIT allowed in subscribe mode\r\n"); - continue 'outer; - } - // Replica: reject writes - if state.is_replica() && is_write_command(&cmd) { - ws_send!(b"-READONLY You can't write against a read only replica.\r\n"); - continue 'outer; - } - // Snapshot commands - match &cmd { - Command::Save => { - state.save(&store).await; - ws_send!(b"+OK\r\n"); - continue 'outer; - } - Command::BgSave => { - let s = Arc::clone(&store); - let st = Arc::clone(&state); - tokio::spawn(async move { st.save(&s).await; }); - ws_send!(b"+Background saving started\r\n"); - continue 'outer; - } - Command::LastSave => { - let ts = state.snap.last_save.load(Ordering::Relaxed); - ws_send!(&Value::Integer(ts).serialize()); - continue 'outer; - } - Command::Info(sections) => { - let repl = ReplInfo { - connected: state.replicas.count.load(Ordering::Relaxed), - queue_depth: state.replicas.max_queue_depth().await, - lag_frames: state.replicas.max_lag_frames().await, - }; - let body = render_info( - sections, - server_facts(), - &store, - sampled_keyspace(&store), - state.is_replica(), - repl, - state.snap.last_save.load(Ordering::Relaxed), - watch_registry.watched_patterns.load(Ordering::Relaxed) as u64, - watch_registry.watched_keys.load(Ordering::Relaxed) as u64, - ); - ws_send!(&Value::BulkString(Some(body.into_bytes())).serialize()); - continue 'outer; - } - Command::Client(args) => { - ws_send!(&handle_client_command(args, &mut client_meta).serialize()); - continue 'outer; - } - Command::Config(args) => { - ws_send!(&handle_config_command(args, server_facts(), &store).serialize()); - continue 'outer; - } - Command::CommandQuery(args) => { - // The WebSocket transport is RESP3-only, - // so the catalog always replies as a map. - ws_send!(&handle_command_query(args, 3).serialize()); - continue 'outer; - } - Command::Cluster(args) => { - ws_send!(&handle_cluster_command(args).serialize()); - continue 'outer; - } - Command::Module(args) => { - ws_send!(&handle_module_command(args).serialize()); - continue 'outer; - } - Command::Memory(args) => { - ws_send!(&handle_memory_command(args).serialize()); - continue 'outer; - } - Command::PubSub(args) => { - ws_send!(&handle_pubsub_command(args, &*pubsub.lock().await).serialize()); - continue 'outer; - } - Command::ReplicaOfNoOne => { - state.promote_to_primary(); - ws_send!(b"+OK\r\n"); - continue 'outer; - } - _ => {} - } - // Ephemeral keys are owned by the connection that wrote - // them until another claims them; the close handler deletes - // whatever is still ours. Claimed outside the write-effects - // branch below, which only runs when a peer, replica, AOF or - // watcher is present — ownership must be recorded even on a - // standalone server with no listeners. - if let Command::ESet(ref k, _) = cmd { - state.claim_ephemeral(k, conn_id); - } - let response = if is_write_command(&cmd) - && write_effects_armed(&tx, &state, &watch_registry) - { - let response = execute_and_record(&store, cmd.clone()); - apply_write_effects(&cmd, &response, &tx, conn_id, &state, &watch_registry, &store).await; - response - } else { - execute_and_record(&store, cmd) - }; - ws_send!(&response.serialize()); - } - } - } - Some(Ok(_)) => {} - Some(Err(e)) => { - warn!("WS error on conn {}: {}", conn_id, e); - break; - } - None => break, - } - } - - result = rx.recv() => { - match result { - Ok(push) if push.origin != conn_id => { - // Scope filter: with scopes set, only matching keys are - // forwarded. Without scopes, legacy mode forwards - // everything; strict mode forwards nothing until a - // token has been presented. - let visible = match &sync_scopes { - Some(scopes) => scopes_match(scopes, &push.keys), - None => !strict, - }; - if visible { - ws_send!(&push.resp); - } - } - Ok(_) => {} - Err(broadcast::error::RecvError::Lagged(n)) => { - warn!("WS conn {} lagged, missed {} messages, resubscribing", conn_id, n); - rx = tx.subscribe(); - } - Err(broadcast::error::RecvError::Closed) => break, - } - } - - msg = ps_rx.recv(), if is_subscribed => { - match msg { - Some(m) => { - let bytes = encode_pubsub_msg(m, 3); - ws_send!(&bytes); - } - None => break, - } - } - - notif = watch_rx.recv(), if !watched_keys.is_empty() => { - if let Some((key, value)) = notif { - // A watched key changed: mark the transaction dirty (so a - // following EXEC aborts) and still push the keychange to the - // client for the observable-keys feature. - watch_dirty = true; - let bytes = encode_keychange(&key, &value); - ws_send!(&bytes); - } - } - - // Live-query keychange: same frame as WATCH pushes, but never - // dirties transactions. - notif = q_rx.recv(), if !qsub_patterns.is_empty() => { - if let Some((key, value)) = notif { - let bytes = encode_keychange(&key, &value); - ws_send!(&bytes); - } - } - } - } - - if !subscribed_channels.is_empty() || !subscribed_patterns.is_empty() { - pubsub.lock().await.unsubscribe_all(conn_id); - } - if !watched_keys.is_empty() { - let mut reg = watch_registry.map.lock().await; - for key in &watched_keys { - if let Some(subs) = reg.get_mut(key) { - subs.retain(|(id, _)| *id != conn_id); - if subs.is_empty() { - reg.remove(key); - } - } - } - watch_registry.sync_len(®); - } - unregister_all_qsubs(&watch_registry, conn_id, &mut qsub_patterns).await; - - // Delete ephemeral keys this connection still owns and fan the deletions - // out, so every subscriber sees the peer go away immediately rather than - // waiting for a heartbeat TTL to lapse. - let expired = state.take_ephemeral_for(conn_id); - if !expired.is_empty() { - let del = Command::Del(expired); - let response = store.execute(del.clone()); - apply_write_effects( - &del, - &response, - &tx, - conn_id, - &state, - &watch_registry, - &store, - ) - .await; - } -} - -#[cfg(test)] -mod tests { - use super::*; - use core_engine::cmd::{ScanArgs, SetOptions, ZAddOptions}; - use core_engine::resp::Value; - use core_engine::store::KeyValueStore; - use std::sync::atomic::{AtomicBool, AtomicI64}; - use tokio::io::{AsyncReadExt, AsyncWriteExt}; - use tokio::net::TcpStream; - - fn tmp_path(name: &str) -> PathBuf { - std::env::temp_dir().join(format!("recached_test_{name}_{}", std::process::id())) - } - - // ── TestServer harness ──────────────────────────────────────────────────── - - struct TestServer { - pub tcp_addr: std::net::SocketAddr, - pub store: Arc, - pub state: Arc, - _task: tokio::task::JoinHandle<()>, - } - - impl Drop for TestServer { - fn drop(&mut self) { - self._task.abort(); - } - } - - async fn spawn_server() -> TestServer { - spawn_server_cfg(None, None, false).await - } - - async fn spawn_server_cfg( - password: Option<&str>, - snap_path: Option, - start_as_replica: bool, - ) -> TestServer { - let store = Arc::new(KeyValueStore::new()); - let (tx, _rx) = broadcast::channel::(256); - let pubsub: SharedPubSub = Arc::new(tokio::sync::Mutex::new(PubSubHub::new())); - let watch_registry: WatchRegistry = WatchHub::new(); - let semaphore = Arc::new(Semaphore::new(64)); - let snap_cfg = Arc::new(SnapshotConfig { - path: snap_path.unwrap_or_else(|| tmp_path("test.rdb")), - last_save: AtomicI64::new(now_unix_secs()), - }); - let state = Arc::new(ServerState { - snap: snap_cfg, - aof: None, - replicas: ReplHub::new(), - is_replica: AtomicBool::new(start_as_replica), - dedup: std::sync::Mutex::new(HashMap::new()), - ephemeral: std::sync::Mutex::new(HashMap::new()), - dedup_dirty: std::sync::atomic::AtomicBool::new(false), - }); - - let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let addr = listener.local_addr().unwrap(); - let store2 = Arc::clone(&store); - let state2 = Arc::clone(&state); - let pass = Arc::new(password.map(|s| s.to_string())); - - let task = tokio::spawn(async move { - loop { - let Ok((socket, _)) = listener.accept().await else { - return; - }; - let Ok(permit) = Arc::clone(&semaphore).try_acquire_owned() else { - continue; - }; - let (s, t, p, ps, wr, st) = ( - Arc::clone(&store2), - tx.clone(), - Arc::clone(&pass), - Arc::clone(&pubsub), - Arc::clone(&watch_registry), - Arc::clone(&state2), - ); - tokio::spawn(async move { - let peer = socket - .peer_addr() - .map(|a| a.to_string()) - .unwrap_or_default(); - handle_tcp(socket, s, t, p, ps, wr, st, peer).await; - drop(permit); - }); - } - }); - - TestServer { - tcp_addr: addr, - store, - state, - _task: task, - } - } - - // ── RespClient ──────────────────────────────────────────────────────────── - - struct RespClient { - stream: TcpStream, - buf: Vec, - filled: usize, - } - - impl RespClient { - async fn connect(addr: std::net::SocketAddr) -> Self { - Self { - stream: TcpStream::connect(addr).await.unwrap(), - buf: vec![0u8; 65536], - filled: 0, - } - } - - /// Send `args` and read one value. An empty `args` sends nothing and - /// just reads the next frame — used to await an out-of-band push. - async fn cmd(&mut self, args: &[&str]) -> Value { - if !args.is_empty() { - let mut req = format!("*{}\r\n", args.len()); - for a in args { - req.push_str(&format!("${}\r\n{}\r\n", a.len(), a)); - } - self.stream.write_all(req.as_bytes()).await.unwrap(); - } - loop { - match Value::parse(&self.buf[..self.filled]) { - Ok((val, n)) => { - self.buf.copy_within(n..self.filled, 0); - self.filled -= n; - return val; - } - Err(e) if e.is_incomplete() => { - let n = self - .stream - .read(&mut self.buf[self.filled..]) - .await - .unwrap(); - assert!(n > 0, "server closed connection unexpectedly"); - self.filled += n; - } - Err(e) => panic!("RESP parse error: {e}"), - } - } - } - - /// True once the peer has closed its half of the connection. - async fn read_raw_eof(&mut self) -> bool { - let mut buf = [0u8; 64]; - matches!(self.stream.read(&mut buf).await, Ok(0)) - } - - async fn read_until_closed(&mut self) { - let mut buf = [0u8; 64]; - while self.stream.read(&mut buf).await.unwrap_or(0) > 0 {} - } - } - - fn ok() -> Value { - Value::SimpleString("OK".to_string()) - } - fn nil() -> Value { - Value::BulkString(None) - } - fn bulk(s: &str) -> Value { - Value::BulkString(Some(s.as_bytes().to_vec())) - } - fn int(n: i64) -> Value { - Value::Integer(n) - } - fn arr(items: &[&str]) -> Value { - Value::Array(Some(items.iter().map(|s| bulk(s)).collect())) - } - - // ── is_write_command ────────────────────────────────────────────────────── - - #[test] - fn is_write_command_classifies_correctly() { - assert!(is_write_command(&Command::Set( - "k".into(), - "v".into(), - SetOptions::default() - ))); - assert!(is_write_command(&Command::Del(vec!["k".into()]))); - assert!(is_write_command(&Command::Incr("k".into()))); - assert!(is_write_command(&Command::FlushDb)); - assert!(is_write_command(&Command::HSet( - "h".into(), - vec![("f".into(), "v".into())] - ))); - assert!(is_write_command(&Command::LPush( - "l".into(), - vec!["v".into()] - ))); - assert!(is_write_command(&Command::SAdd( - "s".into(), - vec!["m".into()] - ))); - assert!(is_write_command(&Command::ZAdd( - "z".into(), - ZAddOptions::default(), - vec![(1.0, "m".into())] - ))); - // reads - assert!(!is_write_command(&Command::Get("k".into()))); - assert!(!is_write_command(&Command::HGet("h".into(), "f".into()))); - assert!(!is_write_command(&Command::LRange("l".into(), 0, -1))); - assert!(!is_write_command(&Command::SMembers("s".into()))); - assert!(!is_write_command(&Command::DbSize)); - assert!(!is_write_command(&Command::Ping(None))); - assert!(!is_write_command(&Command::Publish( - "ch".into(), - "msg".into() - ))); - } - - // ── AOF replay ──────────────────────────────────────────────────────────── - - #[tokio::test] - async fn replay_aof_missing_file() { - let store = KeyValueStore::new(); - let path = tmp_path("aof_missing"); - let count = replay_aof(&store, &path).await; - assert_eq!(count, 0); - } - - #[tokio::test] - async fn replay_aof_basic() { - let store = KeyValueStore::new(); - let path = tmp_path("aof_basic.aof"); - let resp = "*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$3\r\nbar\r\n\ - *3\r\n$3\r\nSET\r\n$3\r\nbaz\r\n$3\r\nqux\r\n"; - tokio::fs::write(&path, resp.as_bytes()).await.unwrap(); - let count = replay_aof(&store, &path).await; - assert_eq!(count, 2); - assert_eq!(store.execute(Command::DbSize), Value::Integer(2)); - let _ = tokio::fs::remove_file(&path).await; - } - - #[tokio::test] - async fn replay_aof_push_frames() { - // The live server records writes via `on_write`, which stores them in - // RESP3 Push (`>`) form. Replay must accept those, not just `*` arrays. - let store = KeyValueStore::new(); - let path = tmp_path("aof_push.aof"); - let resp = ">3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$3\r\nbar\r\n"; - tokio::fs::write(&path, resp.as_bytes()).await.unwrap(); - let count = replay_aof(&store, &path).await; - assert_eq!(count, 1); - assert_eq!( - store.execute(Command::Get("foo".into())), - Value::BulkString(Some(b"bar".to_vec())) - ); - let _ = tokio::fs::remove_file(&path).await; - } - - // ── Snapshot save / load ────────────────────────────────────────────────── - - #[tokio::test] - async fn snapshot_save_and_load() { - let store = KeyValueStore::new(); - store.execute(Command::Set( - "hello".into(), - "world".into(), - SetOptions::default(), - )); - let path = tmp_path("snap.rdb"); - let cfg = Arc::new(SnapshotConfig { - path: path.clone(), - last_save: AtomicI64::new(0), - }); - save_snapshot(&store, &cfg).await; - assert!(path.exists()); - let store2 = KeyValueStore::new(); - let loaded = load_snapshot(&store2, &path).await; - assert!(loaded); - assert_eq!( - store2.execute(Command::Get("hello".into())), - Value::BulkString(Some(b"world".to_vec())) - ); - let _ = tokio::fs::remove_file(&path).await; - } - - // ── AofWriter append / truncate ─────────────────────────────────────────── - - #[tokio::test] - async fn aof_writer_append_and_truncate() { - let path = tmp_path("aof_writer.aof"); - let aof = AofWriter::open(path.clone(), AofSync::No).await.unwrap(); - aof.append(b"*3\r\n$3\r\nSET\r\n$1\r\nk\r\n$1\r\nv\r\n") - .await; - aof.flush().await; - let len_before = tokio::fs::metadata(&path).await.unwrap().len(); - assert!(len_before > 0); - aof.truncate().await; - let len_after = tokio::fs::metadata(&path).await.unwrap().len(); - assert_eq!(len_after, 0); - let _ = tokio::fs::remove_file(&path).await; - } - - // ── Integration: 3a basic commands ─────────────────────────────────────── - - #[tokio::test] - async fn integration_set_get_del() { - let srv = spawn_server().await; - let mut c = RespClient::connect(srv.tcp_addr).await; - - assert_eq!(c.cmd(&["SET", "k", "v"]).await, ok()); - assert_eq!(c.cmd(&["GET", "k"]).await, bulk("v")); - assert_eq!(c.cmd(&["GET", "missing"]).await, nil()); - assert_eq!(c.cmd(&["DEL", "k"]).await, int(1)); - assert_eq!(c.cmd(&["GET", "k"]).await, nil()); - assert_eq!(c.cmd(&["DEL", "k"]).await, int(0)); // already gone - } - - #[tokio::test] - async fn integration_binary_value_round_trips_over_resp() { - // The drop-in claim runs through this port: a value that is not valid - // UTF-8 must come back byte-for-byte, exactly as Redis would. - let srv = spawn_server().await; - let mut c = RespClient::connect(srv.tcp_addr).await; - - let binary: &[u8] = &[0xff, 0xfe, 0x00, 0x41, 0x80]; - let mut req = b"*3\r\n$3\r\nSET\r\n$3\r\nbin\r\n".to_vec(); - req.extend_from_slice(format!("${}\r\n", binary.len()).as_bytes()); - req.extend_from_slice(binary); - req.extend_from_slice(b"\r\n"); - c.stream.write_all(&req).await.unwrap(); - assert_eq!(c.cmd(&[]).await, ok()); - - assert_eq!( - c.cmd(&["GET", "bin"]).await, - Value::BulkString(Some(binary.to_vec())), - "binary value must survive the round trip" - ); - assert_eq!( - c.cmd(&["STRLEN", "bin"]).await, - int(binary.len() as i64), - "length is counted in bytes" - ); - } - - #[tokio::test] - async fn integration_binary_key_is_refused_over_resp() { - // Keys stay text: they are glob-matched and scope-checked, so a - // corrupted one would be silently unreachable. The refusal must be a - // clean RESP error that leaves the connection usable. - let srv = spawn_server().await; - let mut c = RespClient::connect(srv.tcp_addr).await; - - c.stream - .write_all(b"*3\r\n$3\r\nSET\r\n$2\r\n\xff\xfe\r\n$1\r\nv\r\n") - .await - .unwrap(); - let reply = c.cmd(&[]).await; - let Value::Error(e) = &reply else { - panic!("binary key must be refused, got {reply:?}") - }; - assert!(e.contains("must be text"), "error must explain: {e:?}"); - - assert_eq!(c.cmd(&["DBSIZE"]).await, int(0), "nothing may be stored"); - assert_eq!(c.cmd(&["SET", "ok", "v"]).await, ok()); - } - - #[tokio::test] - async fn integration_hello_negotiates_the_protocol() { - let srv = spawn_server().await; - let mut c = RespClient::connect(srv.tcp_addr).await; - - // Default is RESP2: the reply is a flat array, not a map. - let v = c.cmd(&["HELLO"]).await; - let Value::Array(Some(items)) = v else { - panic!("RESP2 HELLO must reply with an array, got {v:?}") - }; - assert!(items.contains(&bulk("recached"))); - assert!(items.contains(&Value::Integer(2))); - - // Upgrading yields a map keyed the same way. - let v = c.cmd(&["HELLO", "3"]).await; - let Value::Map(pairs) = v else { - panic!("RESP3 HELLO must reply with a map, got {v:?}") - }; - let proto = pairs - .iter() - .find(|(k, _)| *k == bulk("proto")) - .map(|(_, v)| v.clone()); - assert_eq!(proto, Some(Value::Integer(3))); - - // An unsupported version is refused and the connection stays usable. - let v = c.cmd(&["HELLO", "9"]).await; - assert!( - matches!(&v, Value::Error(e) if e.starts_with("NOPROTO")), - "expected NOPROTO, got {v:?}" - ); - assert_eq!(c.cmd(&["PING"]).await, Value::SimpleString("PONG".into())); - } - - #[tokio::test] - async fn integration_pubsub_frame_type_follows_the_negotiated_protocol() { - // The bug this pins: pub/sub deliveries were RESP3 push frames on every - // connection, including RESP2 ones that cannot parse `>` at all. - for (protover, want_push) in [(None, false), (Some("3"), true)] { - let srv = spawn_server().await; - let mut sub = RespClient::connect(srv.tcp_addr).await; - if let Some(v) = protover { - sub.cmd(&["HELLO", v]).await; - } - assert!(matches!( - sub.cmd(&["SUBSCRIBE", "news"]).await, - Value::Array(_) | Value::Push(_) - )); - - let mut pubr = RespClient::connect(srv.tcp_addr).await; - // Wait for the subscription to register before publishing. - for _ in 0..50 { - if pubr.cmd(&["PUBLISH", "news", "hi"]).await == int(1) { - break; - } - tokio::time::sleep(tokio::time::Duration::from_millis(20)).await; - } - - let delivery = sub.cmd(&[]).await; - match (&delivery, want_push) { - (Value::Push(_), true) => {} - (Value::Array(Some(_)), false) => {} - _ => panic!("protover {protover:?}: expected push={want_push}, got {delivery:?}"), - } - } - } - - #[tokio::test] - async fn integration_incr_and_expiry() { - let srv = spawn_server().await; - let mut c = RespClient::connect(srv.tcp_addr).await; - - assert_eq!(c.cmd(&["SET", "n", "10"]).await, ok()); - assert_eq!(c.cmd(&["INCR", "n"]).await, int(11)); - assert_eq!(c.cmd(&["INCRBY", "n", "4"]).await, int(15)); - assert_eq!(c.cmd(&["DECR", "n"]).await, int(14)); - - // TTL: set a key with 1-second expiry and verify TTL and eventual expiry - assert_eq!(c.cmd(&["SET", "ex", "val", "EX", "1"]).await, ok()); - let ttl = c.cmd(&["TTL", "ex"]).await; - assert!(matches!(ttl, Value::Integer(1) | Value::Integer(0))); - tokio::time::sleep(tokio::time::Duration::from_millis(1100)).await; - assert_eq!(c.cmd(&["GET", "ex"]).await, nil()); - } - - #[tokio::test] - async fn integration_string_commands() { - let srv = spawn_server().await; - let mut c = RespClient::connect(srv.tcp_addr).await; - - // APPEND + STRLEN - assert_eq!(c.cmd(&["APPEND", "s", "hello"]).await, int(5)); - assert_eq!(c.cmd(&["APPEND", "s", " world"]).await, int(11)); - assert_eq!(c.cmd(&["STRLEN", "s"]).await, int(11)); - - // GETSET - assert_eq!(c.cmd(&["GETSET", "s", "new"]).await, bulk("hello world")); - assert_eq!(c.cmd(&["GET", "s"]).await, bulk("new")); - - // SETNX - assert_eq!(c.cmd(&["SETNX", "nx", "first"]).await, int(1)); - assert_eq!(c.cmd(&["SETNX", "nx", "second"]).await, int(0)); - assert_eq!(c.cmd(&["GET", "nx"]).await, bulk("first")); - - // SETEX - assert_eq!(c.cmd(&["SETEX", "ex", "60", "val"]).await, ok()); - let ttl = c.cmd(&["TTL", "ex"]).await; - assert!(matches!(ttl, Value::Integer(t) if t > 0 && t <= 60)); - - // MSET / MGET - assert_eq!(c.cmd(&["MSET", "a", "1", "b", "2", "c", "3"]).await, ok()); - let got = c.cmd(&["MGET", "a", "b", "c", "missing"]).await; - assert_eq!( - got, - Value::Array(Some(vec![bulk("1"), bulk("2"), bulk("3"), nil()])) - ); - } - - #[tokio::test] - async fn integration_bounded_reads_over_resp() { - // The pair of primitives a client needs to inspect a large key without - // pulling it whole: a byte window into a string, and a cursor over a - // collection. Exercised over the wire because that is where the reply - // shape — bulk cursor, nested array — has to be right. - let srv = spawn_server().await; - let mut c = RespClient::connect(srv.tcp_addr).await; - - c.cmd(&["SET", "s", "This is a string"]).await; - assert_eq!(c.cmd(&["GETRANGE", "s", "0", "3"]).await, bulk("This")); - assert_eq!(c.cmd(&["GETRANGE", "s", "-6", "-1"]).await, bulk("string")); - assert_eq!(c.cmd(&["GETRANGE", "ghost", "0", "-1"]).await, bulk("")); - - c.cmd(&["HSET", "h", "a", "1", "b", "2", "c", "3"]).await; - assert_eq!( - c.cmd(&["HSCAN", "h", "0", "COUNT", "2"]).await, - Value::Array(Some(vec![ - bulk("2"), - Value::Array(Some(vec![bulk("a"), bulk("1"), bulk("b"), bulk("2")])), - ])) - ); - assert_eq!( - c.cmd(&["HSCAN", "h", "2"]).await, - Value::Array(Some(vec![ - bulk("0"), - Value::Array(Some(vec![bulk("c"), bulk("3")])), - ])) - ); - assert_eq!( - c.cmd(&["HSCAN", "h", "0", "NOVALUES"]).await, - Value::Array(Some(vec![ - bulk("0"), - Value::Array(Some(vec![bulk("a"), bulk("b"), bulk("c")])), - ])) - ); - - c.cmd(&["SADD", "st", "x", "y"]).await; - assert_eq!( - c.cmd(&["SSCAN", "st", "0", "MATCH", "x*"]).await, - Value::Array(Some(vec![bulk("0"), Value::Array(Some(vec![bulk("x")])),])) - ); - - c.cmd(&["ZADD", "z", "1.5", "amy"]).await; - assert_eq!( - c.cmd(&["ZSCAN", "z", "0"]).await, - Value::Array(Some(vec![ - bulk("0"), - Value::Array(Some(vec![bulk("amy"), bulk("1.5")])), - ])) - ); - } - - #[tokio::test] - async fn integration_bounded_reads_are_allowed_on_a_replica() { - // Read-only by construction: a replica must serve them, and the - // is_write_command allowlist is what decides that. - let srv = spawn_server_cfg(None, None, true).await; - let mut c = RespClient::connect(srv.tcp_addr).await; - - assert_eq!(c.cmd(&["GETRANGE", "s", "0", "-1"]).await, bulk("")); - assert_eq!( - c.cmd(&["HSCAN", "h", "0"]).await, - Value::Array(Some(vec![bulk("0"), Value::Array(Some(vec![]))])) - ); - } - - #[tokio::test] - async fn integration_handshake_commands_over_resp() { - // Every current client library opens with HELLO + CLIENT SETINFO and - // closes with QUIT. This is that sequence, on the wire. - let srv = spawn_server().await; - let mut c = RespClient::connect(srv.tcp_addr).await; - - assert_eq!( - c.cmd(&["CLIENT", "SETINFO", "LIB-NAME", "node-redis"]) - .await, - ok() - ); - assert_eq!( - c.cmd(&["CLIENT", "SETINFO", "LIB-VER", "6.2.0"]).await, - ok() - ); - assert_eq!(c.cmd(&["CLIENT", "SETNAME", "tap"]).await, ok()); - assert_eq!(c.cmd(&["CLIENT", "GETNAME"]).await, bulk("tap")); - - let Value::Integer(id) = c.cmd(&["CLIENT", "ID"]).await else { - panic!("CLIENT ID must be an integer") - }; - assert!(id > 0); - - let Value::BulkString(Some(info)) = c.cmd(&["CLIENT", "INFO"]).await else { - panic!("CLIENT INFO must be a bulk string") - }; - let info = String::from_utf8(info).unwrap(); - assert!(info.contains(&format!("id={id}")), "{info}"); - assert!(info.contains("lib-name=node-redis"), "{info}"); - assert!(info.contains("name=tap"), "{info}"); - assert!(info.contains("addr=127.0.0.1:"), "{info}"); - - // This connection must appear in the list it asks for. - let Value::BulkString(Some(list)) = c.cmd(&["CLIENT", "LIST"]).await else { - panic!("CLIENT LIST must be a bulk string") - }; - let list = String::from_utf8(list).unwrap(); - assert!( - list.lines().any(|l| l.contains(&format!("id={id}"))), - "{list}" - ); - - assert_eq!( - c.cmd(&["CONFIG", "GET", "maxmemory-policy"]).await, - Value::Array(Some(vec![bulk("maxmemory-policy"), bulk("noeviction")])) - ); - - let Value::Integer(n) = c.cmd(&["COMMAND", "COUNT"]).await else { - panic!("COMMAND COUNT must be an integer") - }; - assert!( - n > 100, - "the catalog should cover the whole command set, got {n}" - ); - } - - #[tokio::test] - async fn integration_quit_replies_then_closes() { - let srv = spawn_server().await; - let mut c = RespClient::connect(srv.tcp_addr).await; - assert_eq!(c.cmd(&["PING"]).await, Value::SimpleString("PONG".into())); - // +OK first, then the close — a client that reads its reply before - // dropping the socket must not see a connection error instead. - assert_eq!(c.cmd(&["QUIT"]).await, ok()); - assert!( - c.read_raw_eof().await, - "the server must close the connection after QUIT" - ); - } - - #[tokio::test] - async fn integration_quit_works_before_authentication() { - // Redis flags QUIT no_auth. A client that cannot authenticate still - // gets a clean close instead of leaving a socket parked on the server. - let srv = spawn_server_cfg(Some("hunter2"), None, false).await; - let mut c = RespClient::connect(srv.tcp_addr).await; - assert!(matches!(c.cmd(&["PING"]).await, Value::Error(e) if e.contains("NOAUTH"))); - assert_eq!(c.cmd(&["QUIT"]).await, ok()); - } - - #[tokio::test] - async fn integration_client_list_sees_other_connections() { - let srv = spawn_server().await; - let mut a = RespClient::connect(srv.tcp_addr).await; - let mut b = RespClient::connect(srv.tcp_addr).await; - b.cmd(&["CLIENT", "SETNAME", "second"]).await; - - let Value::BulkString(Some(list)) = a.cmd(&["CLIENT", "LIST"]).await else { - panic!("expected a bulk string") - }; - let list = String::from_utf8(list).unwrap(); - assert!( - list.lines().any(|l| l.contains("name=second")), - "a connection must see its peers, got:\n{list}" - ); - assert!(list.lines().count() >= 2, "{list}"); - } - - #[tokio::test] - async fn integration_hash_commands() { - let srv = spawn_server().await; - let mut c = RespClient::connect(srv.tcp_addr).await; - - assert_eq!(c.cmd(&["HSET", "h", "f1", "v1", "f2", "v2"]).await, int(2)); - assert_eq!(c.cmd(&["HGET", "h", "f1"]).await, bulk("v1")); - assert_eq!(c.cmd(&["HGET", "h", "missing"]).await, nil()); - assert_eq!(c.cmd(&["HLEN", "h"]).await, int(2)); - assert_eq!(c.cmd(&["HDEL", "h", "f1"]).await, int(1)); - assert_eq!(c.cmd(&["HLEN", "h"]).await, int(1)); - // HGETALL returns field-value pairs - let all = c.cmd(&["HGETALL", "h"]).await; - assert_eq!(all, Value::Array(Some(vec![bulk("f2"), bulk("v2")]))); - } - - #[tokio::test] - async fn integration_list_commands() { - let srv = spawn_server().await; - let mut c = RespClient::connect(srv.tcp_addr).await; - - assert_eq!(c.cmd(&["RPUSH", "l", "a", "b", "c"]).await, int(3)); - assert_eq!(c.cmd(&["LPUSH", "l", "z"]).await, int(4)); - assert_eq!(c.cmd(&["LLEN", "l"]).await, int(4)); - assert_eq!( - c.cmd(&["LRANGE", "l", "0", "-1"]).await, - Value::Array(Some(vec![bulk("z"), bulk("a"), bulk("b"), bulk("c")])) - ); - assert_eq!(c.cmd(&["LPOP", "l"]).await, bulk("z")); - assert_eq!(c.cmd(&["RPOP", "l"]).await, bulk("c")); - assert_eq!(c.cmd(&["LLEN", "l"]).await, int(2)); - } - - #[tokio::test] - async fn integration_set_commands() { - let srv = spawn_server().await; - let mut c = RespClient::connect(srv.tcp_addr).await; - - assert_eq!(c.cmd(&["SADD", "s", "a", "b", "c"]).await, int(3)); - assert_eq!(c.cmd(&["SADD", "s", "a"]).await, int(0)); // duplicate - assert_eq!(c.cmd(&["SCARD", "s"]).await, int(3)); - assert_eq!(c.cmd(&["SISMEMBER", "s", "b"]).await, int(1)); - assert_eq!(c.cmd(&["SISMEMBER", "s", "x"]).await, int(0)); - assert_eq!(c.cmd(&["SREM", "s", "a"]).await, int(1)); - assert_eq!(c.cmd(&["SCARD", "s"]).await, int(2)); - } - - #[tokio::test] - async fn integration_zset_commands() { - let srv = spawn_server().await; - let mut c = RespClient::connect(srv.tcp_addr).await; - - assert_eq!( - c.cmd(&["ZADD", "z", "1.5", "a", "2.5", "b", "3.0", "c"]) - .await, - int(3) - ); - assert_eq!(c.cmd(&["ZCARD", "z"]).await, int(3)); - assert_eq!(c.cmd(&["ZSCORE", "z", "b"]).await, bulk("2.5")); - assert_eq!(c.cmd(&["ZRANK", "z", "a"]).await, int(0)); - assert_eq!(c.cmd(&["ZRANK", "z", "c"]).await, int(2)); - assert_eq!( - c.cmd(&["ZRANGE", "z", "0", "-1", "WITHSCORES"]).await, - Value::Array(Some(vec![ - bulk("a"), - bulk("1.5"), - bulk("b"), - bulk("2.5"), - bulk("c"), - bulk("3"), - ])) - ); - assert_eq!(c.cmd(&["ZREM", "z", "b"]).await, int(1)); - assert_eq!(c.cmd(&["ZCARD", "z"]).await, int(2)); - } - - #[tokio::test] - async fn integration_transactions_exec() { - let srv = spawn_server().await; - let mut c = RespClient::connect(srv.tcp_addr).await; - - assert_eq!(c.cmd(&["SET", "counter", "10"]).await, ok()); - assert_eq!(c.cmd(&["MULTI"]).await, ok()); - assert_eq!( - c.cmd(&["SET", "counter", "20"]).await, - Value::SimpleString("QUEUED".to_string()) - ); - assert_eq!( - c.cmd(&["INCR", "counter"]).await, - Value::SimpleString("QUEUED".to_string()) - ); - let res = c.cmd(&["EXEC"]).await; - assert_eq!(res, Value::Array(Some(vec![ok(), int(21)]))); - assert_eq!(c.cmd(&["GET", "counter"]).await, bulk("21")); - } - - #[tokio::test] - async fn integration_transactions_discard() { - let srv = spawn_server().await; - let mut c = RespClient::connect(srv.tcp_addr).await; - - assert_eq!(c.cmd(&["SET", "key", "original"]).await, ok()); - assert_eq!(c.cmd(&["MULTI"]).await, ok()); - assert_eq!( - c.cmd(&["DEL", "key"]).await, - Value::SimpleString("QUEUED".to_string()) - ); - assert_eq!(c.cmd(&["DISCARD"]).await, ok()); - assert_eq!(c.cmd(&["GET", "key"]).await, bulk("original")); // DEL was discarded - } - - #[tokio::test] - async fn integration_unknown_command() { - let srv = spawn_server().await; - let mut c = RespClient::connect(srv.tcp_addr).await; - - let r = c.cmd(&["NOTACOMMAND", "arg"]).await; - assert!(matches!(r, Value::Error(_))); - } - - // ── Integration: 3b auth ────────────────────────────────────────────────── - - #[tokio::test] - async fn integration_auth_blocks_unauthenticated() { - let srv = spawn_server_cfg(Some("secret"), None, false).await; - let mut c = RespClient::connect(srv.tcp_addr).await; - - let r = c.cmd(&["SET", "k", "v"]).await; - assert!(matches!(&r, Value::Error(e) if e.contains("NOAUTH"))); - } - - #[tokio::test] - async fn integration_auth_correct() { - let srv = spawn_server_cfg(Some("secret"), None, false).await; - let mut c = RespClient::connect(srv.tcp_addr).await; - - assert_eq!(c.cmd(&["AUTH", "secret"]).await, ok()); - assert_eq!(c.cmd(&["SET", "k", "v"]).await, ok()); - assert_eq!(c.cmd(&["GET", "k"]).await, bulk("v")); - } - - #[tokio::test] - async fn integration_auth_wrong_password_lockout() { - let srv = spawn_server_cfg(Some("secret"), None, false).await; - let mut c = RespClient::connect(srv.tcp_addr).await; - - // First 4 wrong attempts → "ERR invalid password" - for _ in 0..4 { - let r = c.cmd(&["AUTH", "wrong"]).await; - assert!(matches!(&r, Value::Error(e) if e.contains("invalid"))); - } - // 5th attempt hits MAX_AUTH_FAILURES → "too many" + server disconnects - let r = c.cmd(&["AUTH", "wrong"]).await; - assert!(matches!(&r, Value::Error(e) if e.contains("too many"))); - c.read_until_closed().await; - } - - // ── Integration: 3c persistence ─────────────────────────────────────────── - - #[tokio::test] - async fn integration_save_and_reload() { - let snap = tmp_path("integ_snap.rdb"); - let srv = spawn_server_cfg(None, Some(snap.clone()), false).await; - let mut c = RespClient::connect(srv.tcp_addr).await; - - assert_eq!(c.cmd(&["SET", "hello", "world"]).await, ok()); - assert_eq!(c.cmd(&["SET", "foo", "bar"]).await, ok()); - assert_eq!(c.cmd(&["SAVE"]).await, ok()); - - // Load into a fresh store - let store2 = KeyValueStore::new(); - let loaded = load_snapshot(&store2, &snap).await; - assert!(loaded); - assert_eq!( - store2.execute(Command::Get("hello".into())), - Value::BulkString(Some(b"world".to_vec())) - ); - assert_eq!( - store2.execute(Command::Get("foo".into())), - Value::BulkString(Some(b"bar".to_vec())) - ); - let _ = tokio::fs::remove_file(&snap).await; - } - - #[tokio::test] - async fn integration_aof_replay() { - let path = tmp_path("integ_aof.aof"); - let aof = AofWriter::open(path.clone(), AofSync::No).await.unwrap(); - let store = KeyValueStore::new(); - let snap_cfg = Arc::new(SnapshotConfig { - path: tmp_path("integ_aof.rdb"), - last_save: AtomicI64::new(0), - }); - let state = Arc::new(ServerState { - snap: snap_cfg, - aof: Some(Arc::new(aof)), - replicas: ReplHub::new(), - is_replica: AtomicBool::new(false), - dedup: std::sync::Mutex::new(HashMap::new()), - ephemeral: std::sync::Mutex::new(HashMap::new()), - dedup_dirty: std::sync::atomic::AtomicBool::new(false), - }); - - // Simulate writes captured by AOF - state - .on_write(b"*3\r\n$3\r\nSET\r\n$5\r\nhello\r\n$5\r\nworld\r\n") - .await; - state - .on_write(b"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$3\r\nbar\r\n") - .await; - if let Some(ref a) = state.aof { - a.flush().await; - } - - // Replay into fresh store - let store2 = KeyValueStore::new(); - let count = replay_aof(&store2, &path).await; - assert_eq!(count, 2); - assert_eq!( - store2.execute(Command::Get("hello".into())), - Value::BulkString(Some(b"world".to_vec())) - ); - drop(store); // suppress unused warning - let _ = tokio::fs::remove_file(&path).await; - } - - #[tokio::test] - async fn integration_dirty_counter() { - let srv = spawn_server().await; - let mut c = RespClient::connect(srv.tcp_addr).await; - - assert_eq!(srv.store.dirty_count(), 0); - - assert_eq!(c.cmd(&["SET", "a", "1"]).await, ok()); - assert_eq!(c.cmd(&["SET", "b", "2"]).await, ok()); - assert_eq!(srv.store.dirty_count(), 2); - - // Trigger a save — dirty resets to 0 - assert_eq!(c.cmd(&["SAVE"]).await, ok()); - assert_eq!(srv.store.dirty_count(), 0); - - // Baseline *after* the explicit save, not before it: SAVE writes - // last_save itself, so a baseline taken beforehand differs by one - // whenever the save lands in the next whole second — which is what this - // test used to fail on, roughly one run in thirty. The assertion below - // is about no *further* save happening. - let last_save = srv.state.snap.last_save.load(Ordering::Relaxed); - - // No new writes → save condition not met → last_save unchanged after 1s - tokio::time::sleep(tokio::time::Duration::from_millis(1100)).await; - assert_eq!( - last_save, - srv.state.snap.last_save.load(Ordering::Relaxed), - "no autosave should fire with no conditions configured" - ); - } - - // ── Integration: 3d replication ─────────────────────────────────────────── - - #[tokio::test] - async fn integration_replica_rejects_writes() { - let srv = spawn_server_cfg(None, None, true).await; - let mut c = RespClient::connect(srv.tcp_addr).await; - - let r = c.cmd(&["SET", "k", "v"]).await; - assert!(matches!(&r, Value::Error(e) if e.contains("READONLY"))); - // Reads still work - assert_eq!(c.cmd(&["GET", "k"]).await, nil()); - } - - #[tokio::test] - async fn integration_replicaof_no_one_promotes() { - let srv = spawn_server_cfg(None, None, true).await; - let mut c = RespClient::connect(srv.tcp_addr).await; - - // Promote - assert_eq!(c.cmd(&["REPLICAOF", "NO", "ONE"]).await, ok()); - // Now writes are accepted - assert_eq!(c.cmd(&["SET", "k", "v"]).await, ok()); - assert_eq!(c.cmd(&["GET", "k"]).await, bulk("v")); - assert!(!srv.state.is_replica()); - } - - // ── FLUSHDB reaches live queries ────────────────────────────────────────── - - /// Collect every frame arriving within `ms`, so a test can assert on the - /// keychange among the command-replay pushes that travel alongside it. - async fn drain_frames(c: &mut WsClient, ms: u64) -> Vec { - let mut out = Vec::new(); - while let Some(f) = c.recv_any(ms).await { - out.push(f); - } - out - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn flushdb_notifies_live_query_subscribers() { - // Previously FLUSHDB emitted nothing to live queries: primary_keys() is - // empty for it, so subscribers kept serving data the server had wiped. - let srv = spawn_ws_server().await; - let mut watcher = WsClient::connect(srv.tcp_addr).await; - watcher.cmd(&["QSUB", "cart:*"]).await; - - let mut writer = WsClient::connect(srv.tcp_addr).await; - writer.cmd(&["SET", "cart:item:1", "a"]).await; - drain_frames(&mut watcher, 400).await; - - writer.cmd(&["FLUSHDB"]).await; - - let frames = drain_frames(&mut watcher, 800).await; - assert!( - frames - .iter() - .any(|f| f.contains("keychange") && f.contains("cart:*")), - "expected a keychange sentinel naming the pattern, got: {frames:?}" - ); - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn flushdb_sends_one_sentinel_per_pattern_not_per_key() { - // The reason for a sentinel: announcing per key would be one frame per - // key in the keyspace for a single command. - let srv = spawn_ws_server().await; - let mut watcher = WsClient::connect(srv.tcp_addr).await; - watcher.cmd(&["QSUB", "bulk:*"]).await; - - let mut writer = WsClient::connect(srv.tcp_addr).await; - for i in 0..25 { - writer.cmd(&["SET", &format!("bulk:{i}"), "v"]).await; - } - drain_frames(&mut watcher, 500).await; - - writer.cmd(&["FLUSHDB"]).await; - - let keychanges: Vec = drain_frames(&mut watcher, 800) - .await - .into_iter() - .filter(|f| f.contains("keychange")) - .collect(); - assert_eq!( - keychanges.len(), - 1, - "25 keys wiped must produce one sentinel, not 25 frames: {keychanges:?}" - ); - assert!(keychanges[0].contains("bulk:*")); - } - - // ── Exactly-once across a restart ───────────────────────────────────────── - - /// Build a `ServerState` whose snapshot path (and therefore dedup sidecar) - /// is `path` — the same file a restarted process would find. - fn state_with_snapshot_path(path: PathBuf) -> Arc { - Arc::new(ServerState { - snap: Arc::new(SnapshotConfig { - path, - last_save: AtomicI64::new(now_unix_secs()), - }), - aof: None, - replicas: ReplHub::new(), - is_replica: AtomicBool::new(false), - dedup: std::sync::Mutex::new(HashMap::new()), - ephemeral: std::sync::Mutex::new(HashMap::new()), - dedup_dirty: std::sync::atomic::AtomicBool::new(false), - }) - } - - #[tokio::test] - async fn dedup_marks_survive_a_restart() { - let snap = tmp_path("dedup_restart.rdb"); - let _ = std::fs::remove_file(snap.with_extension("dedup")); - - // First run: the client's writes are accepted once each. - let first = state_with_snapshot_path(snap.clone()); - assert!(!first.dedup_seen("client-a", 1)); - assert!(!first.dedup_seen("client-a", 2)); - assert!( - first.dedup_seen("client-a", 2), - "same id twice within a run" - ); - first.persist_dedup().await; - - // Restart: fresh process, same snapshot path. - let second = state_with_snapshot_path(snap.clone()); - assert!( - !second.dedup_seen("client-a", 3), - "a genuinely new id must still be accepted" - ); - - let third = state_with_snapshot_path(snap.clone()); - third.load_dedup().await; - assert!( - third.dedup_seen("client-a", 2), - "a replayed write must be recognised after a restart — this is the \ - caveat the sidecar exists to close" - ); - assert!(!third.dedup_seen("client-a", 99), "higher ids still apply"); - - let _ = std::fs::remove_file(snap.with_extension("dedup")); - } - - #[tokio::test] - async fn persist_dedup_is_a_no_op_when_nothing_advanced() { - // The flusher runs every second; it must not rewrite the file when no - // mark moved. - let snap = tmp_path("dedup_noop.rdb"); - let side = snap.with_extension("dedup"); - let _ = std::fs::remove_file(&side); - - let state = state_with_snapshot_path(snap.clone()); - state.dedup_seen("c", 1); - state.persist_dedup().await; - assert!(side.exists(), "first flush should write"); - - let before = std::fs::metadata(&side).unwrap().modified().unwrap(); - state.persist_dedup().await; // nothing changed since - let after = std::fs::metadata(&side).unwrap().modified().unwrap(); - assert_eq!(before, after, "unchanged marks must not rewrite the file"); - - let _ = std::fs::remove_file(&side); - } - - #[tokio::test] - async fn a_corrupt_dedup_sidecar_is_ignored_not_fatal() { - // Losing exactly-once bookkeeping is bad; refusing to boot is worse. - let snap = tmp_path("dedup_corrupt.rdb"); - let side = snap.with_extension("dedup"); - std::fs::write(&side, b"not messagepack").unwrap(); - - let state = state_with_snapshot_path(snap.clone()); - state.load_dedup().await; // must not panic - assert!(!state.dedup_seen("client-a", 1), "server still functions"); - - let _ = std::fs::remove_file(&side); - } - - #[tokio::test] - async fn a_missing_dedup_sidecar_is_a_clean_first_boot() { - let snap = tmp_path("dedup_absent.rdb"); - let _ = std::fs::remove_file(snap.with_extension("dedup")); - let state = state_with_snapshot_path(snap); - state.load_dedup().await; - assert!(!state.dedup_seen("fresh", 1)); - } - - // ── Presence: connection-scoped keys ────────────────────────────────────── - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn eset_key_is_deleted_when_its_connection_closes() { - let srv = spawn_ws_server().await; - let mut watcher = WsClient::connect(srv.tcp_addr).await; - watcher.cmd(&["QSUB", "presence:*"]).await; - - { - let mut presence = WsClient::connect(srv.tcp_addr).await; - assert_eq!( - presence.cmd(&["ESET", "presence:user:42", "online"]).await, - Value::SimpleString("OK".into()) - ); - // Visible to everyone while the connection is open. - assert_eq!( - srv.store.execute(Command::Get("presence:user:42".into())), - Value::BulkString(Some(b"online".to_vec())) - ); - } // connection dropped here - - // The key goes away on its own — no heartbeat, no TTL to wait out. - for _ in 0..40 { - if srv.store.execute(Command::Get("presence:user:42".into())) == Value::BulkString(None) - { - return; - } - tokio::time::sleep(tokio::time::Duration::from_millis(50)).await; - } - panic!("ephemeral key outlived its connection"); - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn eset_deletion_is_broadcast_to_live_queries() { - // Presence is only useful if peers are *told*; polling for the absence - // of a key is the thing this replaces. - let srv = spawn_ws_server().await; - let mut watcher = WsClient::connect(srv.tcp_addr).await; - watcher.cmd(&["QSUB", "presence:*"]).await; - - { - let mut presence = WsClient::connect(srv.tcp_addr).await; - presence.cmd(&["ESET", "presence:user:7", "online"]).await; - // Drain the set notification. - let _ = watcher.recv_push(1000).await; - } - - let frames = drain_frames(&mut watcher, 1500).await; - assert!( - frames - .iter() - .any(|f| f.contains("keychange") && f.contains("presence:user:7")), - "a live query must receive a keychange for the departing peer, got: {frames:?}" - ); - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn a_second_tab_keeps_presence_alive_when_the_first_closes() { - // The multi-tab case. Ownership transfers to the most recent writer, so - // closing an older tab must not mark the user offline. - let srv = spawn_ws_server().await; - - let mut tab_b = WsClient::connect(srv.tcp_addr).await; - { - let mut tab_a = WsClient::connect(srv.tcp_addr).await; - tab_a.cmd(&["ESET", "presence:user:9", "online"]).await; - // Tab B claims the same key — it is now the owner. - tab_b.cmd(&["ESET", "presence:user:9", "online"]).await; - } // tab A closes - - // Give the close handler time to run, then confirm the key survived. - tokio::time::sleep(tokio::time::Duration::from_millis(300)).await; - assert_eq!( - srv.store.execute(Command::Get("presence:user:9".into())), - Value::BulkString(Some(b"online".to_vec())), - "closing an older tab must not clear presence held by a newer one" - ); - - drop(tab_b); - for _ in 0..40 { - if srv.store.execute(Command::Get("presence:user:9".into())) == Value::BulkString(None) - { - return; - } - tokio::time::sleep(tokio::time::Duration::from_millis(50)).await; - } - panic!("key outlived its last owner"); - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn plain_set_is_not_ephemeral() { - // Only ESET opts into connection-scoped lifetime; SET must be unaffected. - let srv = spawn_ws_server().await; - { - let mut c = WsClient::connect(srv.tcp_addr).await; - c.cmd(&["SET", "durable:key", "value"]).await; - } - tokio::time::sleep(tokio::time::Duration::from_millis(300)).await; - assert_eq!( - srv.store.execute(Command::Get("durable:key".into())), - Value::BulkString(Some(b"value".to_vec())) - ); - } - - #[tokio::test] - async fn integration_replica_receives_write() { - // Spawn primary with a separate replication listener on a random port - let primary = spawn_server().await; - let repl_registry: ReplRegistry = ReplHub::new(); - let snap_cfg = Arc::clone(&primary.state.snap); - let primary_store = Arc::clone(&primary.store); - let reg = Arc::clone(&repl_registry); - - // Replication listener — binds on port 0 - let repl_listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let repl_port = repl_listener.local_addr().unwrap().port(); - tokio::spawn(async move { - while let Ok((socket, _)) = repl_listener.accept().await { - let s = Arc::clone(&primary_store); - let sc = Arc::clone(&snap_cfg); - let r = Arc::clone(®); - tokio::spawn(handle_replica( - socket, - s, - sc, - r, - None, - DEFAULT_REPL_CHANNEL_CAPACITY, - IpAddr::from([127, 0, 0, 1]), - ReplAuthThrottle::new(), - )); - } - }); - - // Also wire the repl_registry into the primary state so on_write fans out - // We can't replace state.replicas (it's private), but handle_replica adds - // itself to the registry it receives. We pass the same repl_registry to - // on_write via a workaround: patch primary state's replicas after the fact - // by passing the same Arc. Since ServerState.replicas is private in our - // TestServer, we re-use the one we created. - // ── Simpler approach: replace on_write path by sharing registry ── - // Instead, wire it through the primary ServerState directly. - // (In practice the TestServer shares state.replicas which starts empty; - // handle_replica will push its sender into it when it connects.) - // The trick: we need primary.state.replicas to point to our repl_registry. - // Since TestServer.state is Arc, we can't replace it. - // Use a fresh primary state that shares our registry. - let primary2 = { - let store = Arc::clone(&primary.store); - let (tx, _rx) = broadcast::channel::(256); - let pubsub: SharedPubSub = Arc::new(tokio::sync::Mutex::new(PubSubHub::new())); - let wr: WatchRegistry = WatchHub::new(); - let sem = Arc::new(Semaphore::new(64)); - let snap = Arc::clone(&primary.state.snap); - let state = Arc::new(ServerState { - snap, - aof: None, - replicas: Arc::clone(&repl_registry), - is_replica: AtomicBool::new(false), - dedup: std::sync::Mutex::new(HashMap::new()), - ephemeral: std::sync::Mutex::new(HashMap::new()), - dedup_dirty: std::sync::atomic::AtomicBool::new(false), - }); - let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let addr = listener.local_addr().unwrap(); - let store2 = Arc::clone(&store); - let state2 = Arc::clone(&state); - let pass = Arc::new(None::); - let task = tokio::spawn(async move { - loop { - let Ok((socket, _)) = listener.accept().await else { - return; - }; - let Ok(permit) = Arc::clone(&sem).try_acquire_owned() else { - continue; - }; - let (s, t, p, ps, wrr, st) = ( - Arc::clone(&store2), - tx.clone(), - Arc::clone(&pass), - Arc::clone(&pubsub), - Arc::clone(&wr), - Arc::clone(&state2), - ); - tokio::spawn(async move { - let peer = socket - .peer_addr() - .map(|a| a.to_string()) - .unwrap_or_default(); - handle_tcp(socket, s, t, p, ps, wrr, st, peer).await; - drop(permit); - }); - } - }); - TestServer { - tcp_addr: addr, - store, - state, - _task: task, - } - }; - - // Start replica - let replica_store = Arc::new(KeyValueStore::new()); - let replica_state = Arc::new(ServerState { - snap: Arc::new(SnapshotConfig { - path: tmp_path("repl_snap.rdb"), - last_save: AtomicI64::new(0), - }), - aof: None, - replicas: ReplHub::new(), - is_replica: AtomicBool::new(true), - dedup: std::sync::Mutex::new(HashMap::new()), - ephemeral: std::sync::Mutex::new(HashMap::new()), - dedup_dirty: std::sync::atomic::AtomicBool::new(false), - }); - let rs = Arc::clone(&replica_store); - let rst = Arc::clone(&replica_state); - let repl_addr = format!("127.0.0.1:{repl_port}"); - let rtx = broadcast::channel::(16).0; - tokio::spawn(async move { - run_repl_client(repl_addr, rs, rst, None, None, rtx, None).await; - }); - - // Give replica time to connect and receive initial snapshot - tokio::time::sleep(tokio::time::Duration::from_millis(300)).await; - - // Write to primary2 (which uses the shared repl_registry) - let mut c = RespClient::connect(primary2.tcp_addr).await; - assert_eq!(c.cmd(&["SET", "replkey", "replval"]).await, ok()); - - // Give replication fan-out time to arrive - tokio::time::sleep(tokio::time::Duration::from_millis(150)).await; - - assert_eq!( - replica_store.execute(Command::Get("replkey".into())), - Value::BulkString(Some(b"replval".to_vec())) - ); - - // The replica acknowledges what it applies, so once it has caught up the - // primary must observe zero lag. Before acknowledgements existed the - // primary had no way to distinguish this from a replica that had - // received the frame and silently failed to apply it. - let mut lag = u64::MAX; - for _ in 0..50 { - lag = repl_registry.max_lag_frames().await; - if lag == 0 { - break; - } - tokio::time::sleep(tokio::time::Duration::from_millis(20)).await; - } - assert_eq!(lag, 0, "caught-up replica must report zero lag"); - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn integration_ws_accepts_binary_command_frames() { - // A WebSocket text frame must be well-formed UTF-8, so a command - // carrying raw bytes can only travel in a binary frame. The server - // previously handled text frames only and dropped binary ones. - let srv = spawn_ws_server().await; - let mut c = WsClient::connect(srv.tcp_addr).await; - - // A binary frame whose contents are valid UTF-8 is a normal command. - let raw = b"*3\r\n$3\r\nSET\r\n$3\r\nbin\r\n$5\r\nhello\r\n".to_vec(); - assert_eq!(c.cmd_binary(raw).await, ok()); - assert_eq!(c.cmd(&["GET", "bin"]).await, bulk("hello")); - - // A binary value round-trips byte-for-byte, and the reply comes back in - // a binary frame because it is not valid UTF-8. - let binary: &[u8] = &[0xff, 0xfe, 0x00, 0x41]; - let mut req = b"*3\r\n$3\r\nSET\r\n$3\r\nraw\r\n".to_vec(); - req.extend_from_slice(format!("${}\r\n", binary.len()).as_bytes()); - req.extend_from_slice(binary); - req.extend_from_slice(b"\r\n"); - assert_eq!(c.cmd_binary(req).await, ok()); - assert_eq!( - c.cmd(&["GET", "raw"]).await, - Value::BulkString(Some(binary.to_vec())) - ); - - // The connection stays usable. - assert_eq!(c.cmd(&["PING"]).await, Value::SimpleString("PONG".into())); - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn integration_ws_hello_reports_resp3_and_refuses_downgrade() { - let srv = spawn_ws_server().await; - let mut c = WsClient::connect(srv.tcp_addr).await; - - // The sync protocol is defined in terms of RESP3 push frames. - let v = c.cmd(&["HELLO"]).await; - let Value::Map(pairs) = v else { - panic!("WS HELLO must reply with a RESP3 map, got {v:?}") - }; - assert!( - pairs - .iter() - .any(|(k, v)| *k == bulk("proto") && *v == Value::Integer(3)) - ); - - // Downgrading would silently break push delivery, so it is refused - // rather than accepted-and-ignored. - let v = c.cmd(&["HELLO", "2"]).await; - assert!( - matches!(&v, Value::Error(e) if e.starts_with("NOPROTO")), - "expected NOPROTO, got {v:?}" - ); - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn integration_ws_reaches_the_introspection_commands_too() { - // `handle_tcp` and `handle_ws` are two hand-maintained copies of one - // command loop, so the standing hazard when adding a server-level - // command is wiring it into one and not the other — which compiles, and - // fails only over the transport nobody checked. Every command added - // outside the store belongs in a test like this one. - let srv = spawn_ws_server().await; - let mut c = WsClient::connect(srv.tcp_addr).await; - - assert_eq!(c.cmd(&["SET", "k", "hello"]).await, ok()); - let usage = c.cmd(&["MEMORY", "USAGE", "k"]).await; - assert!( - matches!(usage, Value::Integer(n) if n > 0), - "MEMORY USAGE over WS: {usage:?}" - ); - assert_eq!( - c.cmd(&["MEMORY", "USAGE", "ghost"]).await, - Value::BulkString(None) - ); - assert!(matches!( - c.cmd(&["MEMORY", "DOCTOR"]).await, - Value::Error(_) - )); - - assert_eq!(c.cmd(&["MODULE", "LIST"]).await, Value::Array(Some(vec![]))); - assert!(matches!(c.cmd(&["CLUSTER", "INFO"]).await, Value::Error(_))); - - // No subscribers on this connection, so the registry is empty — the - // point is that the command is answered at all rather than falling - // through to the store's "handled by the connection layer" refusal. - assert_eq!( - c.cmd(&["PUBSUB", "CHANNELS"]).await, - Value::Array(Some(vec![])) - ); - assert_eq!(c.cmd(&["PUBSUB", "NUMPAT"]).await, Value::Integer(0)); - } - - #[tokio::test] - async fn replication_lag_counts_unacknowledged_frames() { - // A replica that receives frames but never acknowledges them is exactly - // the case queue depth cannot see: the frames left the primary's - // channel, so the queue reads empty while the replica is arbitrarily - // far behind. Lag must report them. - let store = Arc::new(KeyValueStore::new()); - let registry: ReplRegistry = ReplHub::new(); - let snap_cfg = Arc::new(SnapshotConfig { - path: tmp_path("lag_snap.rdb"), - last_save: AtomicI64::new(0), - }); - - let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let addr = listener.local_addr().unwrap(); - { - let (s, sc, r) = ( - Arc::clone(&store), - Arc::clone(&snap_cfg), - Arc::clone(®istry), - ); - tokio::spawn(async move { - if let Ok((socket, _)) = listener.accept().await { - let _ = handle_replica( - socket, - s, - sc, - r, - None, - DEFAULT_REPL_CHANNEL_CAPACITY, - IpAddr::from([127, 0, 0, 1]), - ReplAuthThrottle::new(), - ) - .await; - } - }); - } - - // A replica that reads the snapshot and then goes silent. - let mut sock = TcpStream::connect(addr).await.unwrap(); - let mut len_buf = [0u8; 4]; - sock.read_exact(&mut len_buf).await.unwrap(); - let mut snap = vec![0u8; u32::from_le_bytes(len_buf) as usize]; - sock.read_exact(&mut snap).await.unwrap(); - - // Wait for registration, then fan out three writes. - for _ in 0..50 { - if registry.count.load(Ordering::Relaxed) == 1 { - break; - } - tokio::time::sleep(tokio::time::Duration::from_millis(20)).await; - } - for i in 0..3 { - registry - .fan_out(format!("*1\r\n$4\r\nPING{i}\r\n").into_bytes()) - .await; - } - - let mut lag = 0; - for _ in 0..50 { - lag = registry.max_lag_frames().await; - if lag == 3 { - break; - } - tokio::time::sleep(tokio::time::Duration::from_millis(20)).await; - } - assert_eq!(lag, 3, "three unacknowledged frames must show as lag 3"); - - // Acknowledging two of them retires exactly two frames of lag. - sock.write_all(&2u64.to_le_bytes()).await.unwrap(); - for _ in 0..50 { - lag = registry.max_lag_frames().await; - if lag == 1 { - break; - } - tokio::time::sleep(tokio::time::Duration::from_millis(20)).await; - } - assert_eq!(lag, 1, "after acking 2 of 3, one frame remains outstanding"); - - // A stale ack must not walk the high-water mark backwards. - sock.write_all(&1u64.to_le_bytes()).await.unwrap(); - tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; - assert_eq!( - registry.max_lag_frames().await, - 1, - "a replayed lower ack must not increase reported lag" - ); - } - - // ── Integration: 3e load (ignored in normal CI) ─────────────────────────── - - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - #[ignore] - async fn integration_concurrent_writers() { - let srv = Arc::new(spawn_server().await); - let addr = srv.tcp_addr; - - let tasks: Vec<_> = (0..50) - .map(|task_id| { - tokio::spawn(async move { - let mut c = RespClient::connect(addr).await; - for i in 0..100u32 { - let key = format!("t{task_id}_{i}"); - let val = format!("v{i}"); - assert_eq!(c.cmd(&["SET", &key, &val]).await, ok()); - assert_eq!(c.cmd(&["GET", &key]).await, bulk(&val)); - } - }) - }) - .collect(); - - for t in tasks { - t.await.unwrap(); - } - // All 50 × 100 keys should be present - assert_eq!(srv.store.execute(Command::DbSize), Value::Integer(5000)); - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - #[ignore] - async fn integration_connection_limit() { - // Small semaphore: only 3 concurrent connections - let store = Arc::new(KeyValueStore::new()); - let (tx, _rx) = broadcast::channel::(16); - let pubsub: SharedPubSub = Arc::new(tokio::sync::Mutex::new(PubSubHub::new())); - let watch_registry: WatchRegistry = WatchHub::new(); - let semaphore = Arc::new(Semaphore::new(3)); - let state = Arc::new(ServerState { - snap: Arc::new(SnapshotConfig { - path: tmp_path("conn_limit.rdb"), - last_save: AtomicI64::new(0), - }), - aof: None, - replicas: ReplHub::new(), - is_replica: AtomicBool::new(false), - dedup: std::sync::Mutex::new(HashMap::new()), - ephemeral: std::sync::Mutex::new(HashMap::new()), - dedup_dirty: std::sync::atomic::AtomicBool::new(false), - }); - let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let addr = listener.local_addr().unwrap(); - let store2 = Arc::clone(&store); - let state2 = Arc::clone(&state); - let pass = Arc::new(None::); - - tokio::spawn(async move { - loop { - let Ok((socket, _)) = listener.accept().await else { - return; - }; - let Ok(permit) = Arc::clone(&semaphore).try_acquire_owned() else { - // Drop socket immediately — connection limit reached - drop(socket); - continue; - }; - let (s, t, p, ps, wr, st) = ( - Arc::clone(&store2), - tx.clone(), - Arc::clone(&pass), - Arc::clone(&pubsub), - Arc::clone(&watch_registry), - Arc::clone(&state2), - ); - tokio::spawn(async move { - let peer = socket - .peer_addr() - .map(|a| a.to_string()) - .unwrap_or_default(); - handle_tcp(socket, s, t, p, ps, wr, st, peer).await; - drop(permit); - }); - } - }); - - // Open 3 connections and hold them (just send PING and keep the socket open) - let mut holders = Vec::new(); - for _ in 0..3 { - let mut c = RespClient::connect(addr).await; - assert_eq!( - c.cmd(&["PING"]).await, - Value::SimpleString("PONG".to_string()) - ); - holders.push(c); - } - - // 4th connection: server drops it immediately, so read returns 0 - let mut overflow = TcpStream::connect(addr).await.unwrap(); - let mut buf = [0u8; 64]; - let n = overflow.read(&mut buf).await.unwrap_or(0); - assert_eq!(n, 0, "4th connection should have been closed by server"); - - drop(holders); - } - - // ── Integration: 3f chaos (ignored in normal CI) ────────────────────────── - - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - #[ignore] - async fn integration_kill_primary_mid_write() { - let srv = Arc::new(spawn_server().await); - let addr = srv.tcp_addr; - - // Start 20 concurrent writers - let tasks: Vec<_> = (0..20) - .map(|i| { - tokio::spawn(async move { - // Connect; tolerate connection errors (server may die mid-flight) - let stream = TcpStream::connect(addr).await; - if stream.is_err() { - return; - } - let mut c = RespClient { - stream: stream.unwrap(), - buf: vec![0u8; 65536], - filled: 0, - }; - for j in 0..50u32 { - let key = format!("chaos_{i}_{j}"); - // Ignore errors — server may die during this - let _ = tokio::time::timeout( - tokio::time::Duration::from_millis(200), - c.cmd(&["SET", &key, "v"]), - ) - .await; - } - }) - }) - .collect(); - - // Kill the server after 10ms - tokio::time::sleep(tokio::time::Duration::from_millis(10)).await; - srv._task.abort(); - - // Join all writers — none should panic - for t in tasks { - let _ = t.await; - } - - // Store is still intact in memory — no panic is the meaningful assertion here; - // zero keys is valid if the server was killed before any write landed. - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - #[ignore] - async fn integration_failover_promotes() { - // Point replica at a port that refuses connections immediately so the - // unreachable timer starts on the first loop iteration without any - // real primary required. Promotion happens after: - // connect fail (fast) → backoff 2s → connect fail → elapsed ≥ 1s → promote - // so we wait 3s to be safe. - let replica_state = Arc::new(ServerState { - snap: Arc::new(SnapshotConfig { - path: tmp_path("failover_snap.rdb"), - last_save: AtomicI64::new(0), - }), - aof: None, - replicas: ReplHub::new(), - is_replica: AtomicBool::new(true), - dedup: std::sync::Mutex::new(HashMap::new()), - ephemeral: std::sync::Mutex::new(HashMap::new()), - dedup_dirty: std::sync::atomic::AtomicBool::new(false), - }); - let replica_store = Arc::new(KeyValueStore::new()); - let rs = Arc::clone(&replica_store); - let rst = Arc::clone(&replica_state); - // Bind a listener then immediately drop it so the port is known-refused - let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let dead_addr = format!("127.0.0.1:{}", listener.local_addr().unwrap().port()); - drop(listener); - let rtx = broadcast::channel::(16).0; - tokio::spawn(async move { - run_repl_client(dead_addr, rs, rst, None, Some(1), rtx, None).await; - }); - - // Wait for 2 backoff cycles (initial fail + 2s sleep + retry fail → promote) - tokio::time::sleep(tokio::time::Duration::from_millis(3000)).await; - - assert!( - !replica_state.is_replica(), - "replica should have promoted after primary was unreachable for >1s" - ); - } - - // ── WebSocket WATCH/EXEC harness ────────────────────────────────────────── - - /// Spawn a WebSocket server sharing one store + watch registry across all - /// connections, so WATCH notifications fan out between clients. - async fn spawn_ws_server() -> TestServer { - spawn_ws_server_cfg(None).await - } - - /// Like `spawn_ws_server`, with an optional sync-scope secret (strict mode). - async fn spawn_ws_server_cfg(sync_secret: Option) -> TestServer { - spawn_ws_server_full(sync_secret, None).await - } - - /// Like `spawn_ws_server`, with an origin allowlist in force. - async fn spawn_ws_server_origins(origins: Vec) -> TestServer { - spawn_ws_server_full(None, Some(origins)).await - } - - /// Open a WebSocket to `addr`, optionally sending an `Origin` header, and - /// report whether the handshake completed. - async fn ws_connect_with_origin( - addr: std::net::SocketAddr, - origin: Option<&str>, - ) -> Result<(), String> { - use tokio_tungstenite::tungstenite::client::IntoClientRequest; - let mut req = format!("ws://{addr}").into_client_request().unwrap(); - if let Some(o) = origin { - req.headers_mut().insert("origin", o.parse().unwrap()); - } - tokio_tungstenite::connect_async(req) - .await - .map(|_| ()) - .map_err(|e| e.to_string()) - } - - #[tokio::test] - async fn ws_refuses_a_cross_origin_handshake() { - // Browsers apply neither CORS nor a preflight to WebSockets, so before - // this check any page a user visited could open a socket to port 6380 - // and read or write the whole keyspace with that user's network - // position. On ws://localhost:6380 that is every site in every tab. - let srv = spawn_ws_server_origins(vec!["https://app.example.com".to_string()]).await; - - let err = ws_connect_with_origin(srv.tcp_addr, Some("https://evil.example")) - .await - .expect_err("a foreign Origin must not complete the handshake"); - assert!( - err.contains("403") || err.to_lowercase().contains("forbidden"), - "expected a 403, got {err}" - ); - } - - #[tokio::test] - async fn ws_admits_an_allowlisted_origin_and_serves_commands() { - // The rejection is worthless if it also breaks the deployed app, so - // assert the permitted path all the way through to a working command. - let srv = spawn_ws_server_origins(vec!["https://app.example.com".to_string()]).await; - ws_connect_with_origin(srv.tcp_addr, Some("https://app.example.com")) - .await - .expect("an allowlisted Origin must connect"); - - let mut c = WsClient::connect(srv.tcp_addr).await; - assert_eq!(c.cmd(&["SET", "k", "v"]).await, ok()); - assert_eq!(c.cmd(&["GET", "k"]).await, bulk("v")); - } - - #[tokio::test] - async fn ws_admits_a_client_that_sends_no_origin() { - // Native clients omit the header, and an attacker with a raw socket can - // forge it — refusing here would break real clients while stopping - // nobody. `WsClient::connect` is exactly such a client. - let srv = spawn_ws_server_origins(vec!["https://app.example.com".to_string()]).await; - ws_connect_with_origin(srv.tcp_addr, None) - .await - .expect("a client with no Origin must connect"); - } - - #[tokio::test] - async fn ws_without_an_allowlist_accepts_any_origin() { - // Unset means allow, matching RECACHED_PASSWORD. The startup warning is - // what keeps this from being a silent default. - let srv = spawn_ws_server().await; - ws_connect_with_origin(srv.tcp_addr, Some("https://anything.example")) - .await - .expect("no allowlist means no origin restriction"); - } - - async fn spawn_ws_server_full( - sync_secret: Option, - allowed_origins: Option>, - ) -> TestServer { - let store = Arc::new(KeyValueStore::new()); - let (tx, _rx) = broadcast::channel::(256); - let pubsub: SharedPubSub = Arc::new(tokio::sync::Mutex::new(PubSubHub::new())); - let watch_registry: WatchRegistry = WatchHub::new(); - let snap_cfg = Arc::new(SnapshotConfig { - path: tmp_path("ws_test.rdb"), - last_save: AtomicI64::new(now_unix_secs()), - }); - let state = Arc::new(ServerState { - snap: snap_cfg, - aof: None, - replicas: ReplHub::new(), - is_replica: AtomicBool::new(false), - dedup: std::sync::Mutex::new(HashMap::new()), - ephemeral: std::sync::Mutex::new(HashMap::new()), - dedup_dirty: std::sync::atomic::AtomicBool::new(false), - }); - - let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let addr = listener.local_addr().unwrap(); - let store2 = Arc::clone(&store); - let state2 = Arc::clone(&state); - let secret = Arc::new(sync_secret); - let origins = Arc::new(allowed_origins); - - let task = tokio::spawn(async move { - loop { - let Ok((socket, _)) = listener.accept().await else { - return; - }; - let (s, t, ps, wr, st, ss, ao) = ( - Arc::clone(&store2), - tx.clone(), - Arc::clone(&pubsub), - Arc::clone(&watch_registry), - Arc::clone(&state2), - Arc::clone(&secret), - Arc::clone(&origins), - ); - let id = next_conn_id(); - let peer = socket - .peer_addr() - .map(|a| a.to_string()) - .unwrap_or_default(); - tokio::spawn(async move { - handle_ws(socket, s, t, Arc::new(None), id, ps, wr, st, ss, ao, peer).await; - }); - } - }); - - TestServer { - tcp_addr: addr, - store, - state, - _task: task, - } - } - - struct WsClient { - ws: tokio_tungstenite::WebSocketStream>, - } - - impl WsClient { - async fn connect(addr: std::net::SocketAddr) -> Self { - let (ws, _) = tokio_tungstenite::connect_async(format!("ws://{addr}")) - .await - .unwrap(); - Self { ws } - } - - async fn cmd(&mut self, args: &[&str]) -> Value { - let mut req = format!("*{}\r\n", args.len()); - for a in args { - req.push_str(&format!("${}\r\n{}\r\n", a.len(), a)); - } - self.ws.send(Message::Text(req.into())).await.unwrap(); - self.next_reply().await - } - - /// Send pre-encoded RESP bytes in a *binary* frame. Text frames must be - /// well-formed UTF-8 per the WebSocket spec, so this is the only way to - /// put arbitrary bytes on the wire. - async fn cmd_binary(&mut self, raw: Vec) -> Value { - self.ws.send(Message::Binary(raw.into())).await.unwrap(); - self.next_reply().await - } - - /// Wait up to `ms` for the next frame of any kind — RESP3 Push - /// broadcasts *and* plain arrays. `keychange` notifications are encoded - /// as arrays, so `recv_push` skips them entirely. - async fn recv_any(&mut self, ms: u64) -> Option { - let fut = async { - loop { - match self.ws.next().await { - Some(Ok(Message::Text(t))) => return Some(t.to_string()), - Some(Ok(_)) => continue, - _ => return None, - } - } - }; - tokio::time::timeout(tokio::time::Duration::from_millis(ms), fut) - .await - .ok() - .flatten() - } - - /// Wait up to `ms` for the next RESP3 Push broadcast frame, returning - /// its raw text. `None` when nothing arrives in time. - async fn recv_push(&mut self, ms: u64) -> Option { - let fut = async { - loop { - match self.ws.next().await { - Some(Ok(Message::Text(t))) => { - let Ok((v, _)) = Value::parse(t.as_bytes()) else { - continue; - }; - if matches!(v, Value::Push(_)) { - return Some(t.to_string()); - } - } - Some(Ok(_)) => continue, - _ => return None, - } - } - }; - tokio::time::timeout(tokio::time::Duration::from_millis(ms), fut) - .await - .ok() - .flatten() - } - - /// Wait up to `ms` for the next `keychange` frame (WATCH / live-query - /// push), returning `(key, value)`. `None` when nothing arrives. - async fn recv_keychange(&mut self, ms: u64) -> Option<(String, Value)> { - let fut = async { - loop { - match self.ws.next().await { - Some(Ok(Message::Text(t))) => { - let Ok((v, _)) = Value::parse(t.as_bytes()) else { - continue; - }; - if let Value::Array(Some(items)) = &v - && items.len() == 3 - && matches!(items.first(), Some(Value::BulkString(Some(k))) if k == b"keychange") - { - let Value::BulkString(Some(key)) = &items[1] else { - continue; - }; - return Some(( - String::from_utf8_lossy(key).into_owned(), - items[2].clone(), - )); - } - } - Some(Ok(_)) => continue, - _ => return None, - } - } - }; - tokio::time::timeout(tokio::time::Duration::from_millis(ms), fut) - .await - .ok() - .flatten() - } - - /// Read the next *command reply*, skipping server-initiated frames - /// (RESP3 Push broadcasts and `keychange` observable-key pushes). - async fn next_reply(&mut self) -> Value { - loop { - let raw: Vec = match self.ws.next().await { - Some(Ok(Message::Text(t))) => t.as_bytes().to_vec(), - Some(Ok(Message::Binary(b))) => b.to_vec(), - Some(Ok(_)) => continue, - _ => panic!("ws closed unexpectedly"), - }; - let Ok((v, _)) = Value::parse(&raw) else { - continue; - }; - if matches!(v, Value::Push(_)) { - continue; - } - if let Value::Array(Some(items)) = &v - && matches!(items.first(), Some(Value::BulkString(Some(k))) if k == b"keychange") - { - continue; - } - return v; - } - } - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn integration_ws_watch_exec_aborts_on_change() { - let srv = spawn_ws_server().await; - let mut watcher = WsClient::connect(srv.tcp_addr).await; - let mut writer = WsClient::connect(srv.tcp_addr).await; - - assert_eq!(watcher.cmd(&["SET", "k", "v0"]).await, ok()); - assert_eq!(watcher.cmd(&["WATCH", "k"]).await, ok()); - - // Another client mutates the watched key. - assert_eq!(writer.cmd(&["SET", "k", "v1"]).await, ok()); - // Give the notification time to reach the watcher's registry channel. - tokio::time::sleep(tokio::time::Duration::from_millis(50)).await; - - assert_eq!( - watcher.cmd(&["MULTI"]).await, - Value::SimpleString("OK".into()) - ); - assert_eq!( - watcher.cmd(&["SET", "k", "v2"]).await, - Value::SimpleString("QUEUED".into()) - ); - // EXEC must abort with a nil array because k changed since WATCH. - assert_eq!(watcher.cmd(&["EXEC"]).await, Value::Array(None)); - // The transaction did not run. - assert_eq!(srv.store.execute(Command::Get("k".into())), bulk("v1")); - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn integration_ws_watch_exec_runs_when_unchanged() { - let srv = spawn_ws_server().await; - let mut c = WsClient::connect(srv.tcp_addr).await; - - assert_eq!(c.cmd(&["WATCH", "k"]).await, ok()); - assert_eq!(c.cmd(&["MULTI"]).await, ok()); - assert_eq!( - c.cmd(&["SET", "k", "v1"]).await, - Value::SimpleString("QUEUED".into()) - ); - // No one touched k → EXEC runs and returns the queued results. - assert_eq!( - c.cmd(&["EXEC"]).await, - Value::Array(Some(vec![Value::SimpleString("OK".into())])) - ); - assert_eq!(srv.store.execute(Command::Get("k".into())), bulk("v1")); - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn integration_tcp_watch_exec_aborts_on_change() { - let srv = spawn_server().await; - let mut watcher = RespClient::connect(srv.tcp_addr).await; - let mut writer = RespClient::connect(srv.tcp_addr).await; - - assert_eq!(watcher.cmd(&["SET", "k", "v0"]).await, ok()); - assert_eq!(watcher.cmd(&["WATCH", "k"]).await, ok()); - // Another client mutates the watched key (reply awaited → notification queued). - assert_eq!(writer.cmd(&["SET", "k", "v1"]).await, ok()); - - assert_eq!(watcher.cmd(&["MULTI"]).await, ok()); - assert_eq!( - watcher.cmd(&["SET", "k", "v2"]).await, - Value::SimpleString("QUEUED".into()) - ); - // k changed since WATCH → EXEC aborts with a nil array. - assert_eq!(watcher.cmd(&["EXEC"]).await, Value::Array(None)); - assert_eq!(watcher.cmd(&["GET", "k"]).await, bulk("v1")); - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn integration_tcp_watch_exec_runs_when_unchanged() { - let srv = spawn_server().await; - let mut c = RespClient::connect(srv.tcp_addr).await; - - assert_eq!(c.cmd(&["WATCH", "k"]).await, ok()); - assert_eq!(c.cmd(&["MULTI"]).await, ok()); - assert_eq!( - c.cmd(&["SET", "k", "v1"]).await, - Value::SimpleString("QUEUED".into()) - ); - assert_eq!( - c.cmd(&["EXEC"]).await, - Value::Array(Some(vec![Value::SimpleString("OK".into())])) - ); - assert_eq!(c.cmd(&["GET", "k"]).await, bulk("v1")); - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn integration_tcp_watch_inside_multi_rejected() { - let srv = spawn_server().await; - let mut c = RespClient::connect(srv.tcp_addr).await; - assert_eq!(c.cmd(&["MULTI"]).await, ok()); - // WATCH is not allowed once a transaction has started. - assert!(matches!(c.cmd(&["WATCH", "k"]).await, Value::Error(_))); - } - - // ── Sync scoping ────────────────────────────────────────────────────────── - - /// Mint a sync-scope token the way an application backend would: - /// HMAC-SHA256 over the base64url payload text. - fn mint_sync_token(secret: &str, payload: &str) -> String { - use base64::Engine as _; - use hmac::{Hmac, Mac}; - let engine = base64::engine::general_purpose::URL_SAFE_NO_PAD; - let payload_b64 = engine.encode(payload); - let mut mac = Hmac::::new_from_slice(secret.as_bytes()).unwrap(); - mac.update(payload_b64.as_bytes()); - let sig = engine.encode(mac.finalize().into_bytes()); - format!("{payload_b64}.{sig}") - } - - #[test] - fn sync_token_roundtrip_and_rejections() { - let tok = mint_sync_token("s3cret", "cart:42:*,profile:42"); - assert_eq!( - verify_sync_token("s3cret", &tok).unwrap(), - vec!["cart:42:*".to_string(), "profile:42".to_string()] - ); - // Wrong secret → invalid signature. - assert_eq!( - verify_sync_token("other", &tok).unwrap_err(), - "invalid signature" - ); - // Tampered payload → invalid signature. - let (_, sig) = tok.split_once('.').unwrap(); - use base64::Engine as _; - let engine = base64::engine::general_purpose::URL_SAFE_NO_PAD; - let forged = format!("{}.{}", engine.encode("admin:*"), sig); - assert_eq!( - verify_sync_token("s3cret", &forged).unwrap_err(), - "invalid signature" - ); - // Expired token. - let expired = mint_sync_token("s3cret", "cart:*|1"); - assert_eq!( - verify_sync_token("s3cret", &expired).unwrap_err(), - "token expired" - ); - // Future expiry still valid. - let live = mint_sync_token("s3cret", "cart:*|99999999999"); - assert!(verify_sync_token("s3cret", &live).is_ok()); - // Empty patterns / malformed. - let empty = mint_sync_token("s3cret", ""); - assert_eq!( - verify_sync_token("s3cret", &empty).unwrap_err(), - "token grants no patterns" - ); - assert!(verify_sync_token("s3cret", "no-dot-here").is_err()); - } - - #[test] - fn scopes_match_globs_and_flushdb() { - let scopes = vec!["cart:42:*".to_string(), "catalog:*".to_string()]; - assert!(scopes_match(&scopes, &["cart:42:item:1".to_string()])); - assert!(scopes_match(&scopes, &["catalog:books".to_string()])); - assert!(!scopes_match(&scopes, &["cart:7:item:1".to_string()])); - assert!(!scopes_match(&scopes, &["session:42".to_string()])); - // Multi-key: any matching key makes the push visible. - assert!(scopes_match( - &scopes, - &["session:42".to_string(), "catalog:books".to_string()] - )); - // No keys = FLUSHDB — visible to every scope. - assert!(scopes_match(&scopes, &[])); - } - - #[test] - fn command_scope_classification() { - assert!(matches!( - command_scope(&Command::Ping(None)), - CommandScope::KeyLess - )); - assert!(matches!( - command_scope(&Command::Keys("*".into())), - CommandScope::Admin - )); - assert!(matches!( - command_scope(&Command::FlushDb), - CommandScope::Admin - )); - match command_scope(&Command::Get("a".into())) { - CommandScope::Keys(k) => assert_eq!(k, vec!["a".to_string()]), - _ => panic!("GET should be key-scoped"), - } - match command_scope(&Command::SInterStore( - "dst".into(), - vec!["a".into(), "b".into()], - )) { - CommandScope::Keys(k) => { - assert!(k.contains(&"dst".to_string()) && k.contains(&"a".to_string())) - } - _ => panic!("SINTERSTORE should be key-scoped"), - } - } - - // ── Scope enforcement: the authorization surface ────────────────────────── - // - // `command_scope` decides whether a command is checked against the - // connection's grants at all. A key-touching command misclassified as - // `KeyLess` skips the check entirely, so every family is pinned here. - // The match in `command_scope` is exhaustive over `Command` with no - // catch-all — a new variant fails to compile until classified — and these - // tests guard against the remaining risk: classifying one wrongly. - - fn scoped_keys(cmd: Command) -> Vec { - match command_scope(&cmd) { - CommandScope::Keys(k) => k, - other => panic!("{cmd:?} should be key-scoped, got {other:?}"), - } - } - - #[test] - fn every_single_key_command_family_is_key_scoped() { - let cases: Vec<(Command, &str)> = vec![ - (Command::Get("k".into()), "k"), - ( - Command::Set("k".into(), "v".into(), SetOptions::default()), - "k", - ), - (Command::Append("k".into(), "v".into()), "k"), - (Command::Incr("k".into()), "k"), - (Command::Decr("k".into()), "k"), - (Command::Ttl("k".into()), "k"), - (Command::Persist("k".into()), "k"), - (Command::Expire("k".into(), 1), "k"), - (Command::Type("k".into()), "k"), - (Command::HGet("k".into(), "f".into()), "k"), - (Command::HGetAll("k".into()), "k"), - (Command::LPush("k".into(), vec!["v".into()]), "k"), - (Command::LRange("k".into(), 0, -1), "k"), - (Command::SAdd("k".into(), vec!["m".into()]), "k"), - (Command::SMembers("k".into()), "k"), - ( - Command::ZAdd("k".into(), ZAddOptions::default(), vec![(1.0, "m".into())]), - "k", - ), - (Command::ZScore("k".into(), "m".into()), "k"), - (Command::JGet("k".into(), None), "k"), - (Command::JSet("k".into(), "$".into(), "1".into()), "k"), - (Command::RlCheck("k".into(), None), "k"), - ]; - for (cmd, expected) in cases { - let keys = scoped_keys(cmd.clone()); - assert!( - keys.contains(&expected.to_string()), - "{cmd:?} must report key '{expected}', got {keys:?}" - ); - } - } - - #[test] - fn multi_key_commands_report_every_key_they_touch() { - // A key omitted here is a key that never gets scope-checked, because - // `scopes_match` only inspects the keys it is handed. - let rename = scoped_keys(Command::Rename("src".into(), "dst".into())); - assert!( - rename.contains(&"src".to_string()) && rename.contains(&"dst".to_string()), - "RENAME touches both keys, got {rename:?}" - ); - - let smove = scoped_keys(Command::SMove("src".into(), "dst".into(), "m".into())); - assert!( - smove.contains(&"src".to_string()) && smove.contains(&"dst".to_string()), - "SMOVE touches both sets, got {smove:?}" - ); - - let mset = scoped_keys(Command::MSet(vec![ - ("a".into(), "1".into()), - ("b".into(), "2".into()), - ])); - assert!( - mset.contains(&"a".to_string()) && mset.contains(&"b".to_string()), - "MSET touches every key, got {mset:?}" - ); - - let mget = scoped_keys(Command::MGet(vec!["a".into(), "b".into()])); - assert!(mget.contains(&"a".to_string()) && mget.contains(&"b".to_string())); - - let del = scoped_keys(Command::Del(vec!["a".into(), "b".into()])); - assert!(del.contains(&"a".to_string()) && del.contains(&"b".to_string())); - - let exists = scoped_keys(Command::Exists(vec!["a".into(), "b".into()])); - assert!(exists.contains(&"a".to_string()) && exists.contains(&"b".to_string())); - - let sunion = scoped_keys(Command::SUnionStore( - "dst".into(), - vec!["a".into(), "b".into()], - )); - for k in ["dst", "a", "b"] { - assert!(sunion.contains(&k.to_string()), "SUNIONSTORE missed {k}"); - } - } - - #[test] - fn administrative_commands_are_denied_not_merely_unscoped() { - // These would read or destroy data outside any grant, so they must map - // to Admin (refused) rather than KeyLess (silently allowed). - for cmd in [ - Command::Keys("*".into()), - Command::Scan(0, None, None), - Command::DbSize, - Command::FlushDb, - Command::Save, - Command::BgSave, - Command::LastSave, - Command::ReplicaOfNoOne, - ] { - assert!( - matches!(command_scope(&cmd), CommandScope::Admin), - "{cmd:?} must be Admin-classified" - ); - } - } - - #[test] - fn keyless_commands_touch_no_keys() { - for cmd in [ - Command::Ping(None), - Command::Auth("pw".into()), - Command::Multi, - Command::Exec, - Command::Discard, - Command::Subscribe(vec!["ch".into()]), - Command::Publish("ch".into(), "m".into()), - Command::Sync(vec![]), - ] { - assert!( - matches!(command_scope(&cmd), CommandScope::KeyLess), - "{cmd:?} should be KeyLess" - ); - } - } - - #[test] - fn dedup_envelope_inherits_the_inner_command_scope() { - // DEDUP wraps a real write. If the wrapper were treated as KeyLess, an - // attacker could smuggle any command past scope enforcement. - let inner = Command::Set("secret:1".into(), "v".into(), SetOptions::default()); - let wrapped = Command::Dedup("client".into(), 1, Box::new(inner)); - match command_scope(&wrapped) { - CommandScope::Keys(k) => assert_eq!(k, vec!["secret:1".to_string()]), - other => panic!("DEDUP must inherit inner scope, got {other:?}"), - } - - // The same must hold for an admin inner command. - let wrapped_admin = Command::Dedup("client".into(), 2, Box::new(Command::FlushDb)); - assert!(matches!(command_scope(&wrapped_admin), CommandScope::Admin)); - } - - #[test] - fn scopes_match_allows_when_there_are_no_keys_to_check() { - // Documented consequence of the design: an empty key list is allowed. - // That is only safe because `command_scope` returns `Keys(..)` for every - // key-touching command — the test above is what keeps it safe. - assert!(scopes_match(&["cart:*".to_string()], &[])); - } - - #[test] - fn scopes_match_requires_every_key_to_match() { - let scopes = vec!["cart:42:*".to_string()]; - assert!(scopes_match(&scopes, &["cart:42:a".to_string()])); - // One in-scope key does not license an out-of-scope sibling. - assert!(!scopes_match(&scopes, &["cart:99:a".to_string()])); - } - - // ── Metrics labels ──────────────────────────────────────────────────────── - - // ── Exhaustive command classification ───────────────────────────────────── - // - // One instance of every `Command` variant, run through the three functions - // that decide scope enforcement and metrics. `command_scope` has no - // catch-all arm, so a new variant cannot compile without being classified — - // but nothing stops it being classified *wrongly*, and a key-touching - // command marked `KeyLess` silently bypasses scope checks entirely. - - enum Expect { - KeyLess, - Admin, - Keys(&'static [&'static str]), - } - - fn all_commands() -> Vec<(Command, Expect)> { - vec![ - (Command::Ping(None), Expect::KeyLess), - (Command::Auth("pw".into()), Expect::KeyLess), - ( - Command::Set("k".into(), "v".into(), SetOptions::default()), - Expect::Keys(&["k"]), - ), - (Command::Get("k".into()), Expect::Keys(&["k"])), - (Command::ESet("k".into(), "v".into()), Expect::Keys(&["k"])), - (Command::Del(vec!["k".into()]), Expect::Keys(&["k"])), - (Command::Unlink(vec!["k".into()]), Expect::Keys(&["k"])), - ( - Command::Append("k".into(), "v".into()), - Expect::Keys(&["k"]), - ), - (Command::Strlen("k".into()), Expect::Keys(&["k"])), - (Command::GetRange("k".into(), 0, -1), Expect::Keys(&["k"])), - ( - Command::GetSet("k".into(), "v".into()), - Expect::Keys(&["k"]), - ), - (Command::MGet(vec!["k".into()]), Expect::Keys(&["k"])), - (Command::SetNx("k".into(), "v".into()), Expect::Keys(&["k"])), - ( - Command::SetEx("k".into(), 1, "v".into()), - Expect::Keys(&["k"]), - ), - ( - Command::PSetEx("k".into(), 1, "v".into()), - Expect::Keys(&["k"]), - ), - ( - Command::MSet(vec![("k".into(), "v".into())]), - Expect::Keys(&["k"]), - ), - (Command::Incr("k".into()), Expect::Keys(&["k"])), - (Command::Decr("k".into()), Expect::Keys(&["k"])), - (Command::IncrBy("k".into(), 1), Expect::Keys(&["k"])), - (Command::DecrBy("k".into(), 1), Expect::Keys(&["k"])), - (Command::Expire("k".into(), 1), Expect::Keys(&["k"])), - (Command::PExpire("k".into(), 1), Expect::Keys(&["k"])), - (Command::ExpireAt("k".into(), 1), Expect::Keys(&["k"])), - (Command::PExpireAt("k".into(), 1), Expect::Keys(&["k"])), - (Command::Ttl("k".into()), Expect::Keys(&["k"])), - (Command::PTtl("k".into()), Expect::Keys(&["k"])), - (Command::Persist("k".into()), Expect::Keys(&["k"])), - (Command::Exists(vec!["k".into()]), Expect::Keys(&["k"])), - (Command::Keys("*".into()), Expect::Admin), - (Command::Scan(0, None, None), Expect::Admin), - (Command::DbSize, Expect::Admin), - (Command::FlushDb, Expect::Admin), - ( - Command::Rename("k".into(), "d".into()), - Expect::Keys(&["k", "d"]), - ), - (Command::Type("k".into()), Expect::Keys(&["k"])), - ( - Command::HSet("k".into(), vec![("f".into(), "v".into())]), - Expect::Keys(&["k"]), - ), - (Command::HGet("k".into(), "f".into()), Expect::Keys(&["k"])), - (Command::HGetAll("k".into()), Expect::Keys(&["k"])), - ( - Command::HDel("k".into(), vec!["f".into()]), - Expect::Keys(&["k"]), - ), - (Command::HKeys("k".into()), Expect::Keys(&["k"])), - (Command::HVals("k".into()), Expect::Keys(&["k"])), - (Command::HLen("k".into()), Expect::Keys(&["k"])), - ( - Command::HIncrBy("k".into(), "f".into(), 1), - Expect::Keys(&["k"]), - ), - ( - Command::HIncrByFloat("k".into(), "f".into(), 1.0), - Expect::Keys(&["k"]), - ), - ( - Command::HExists("k".into(), "f".into()), - Expect::Keys(&["k"]), - ), - ( - Command::HSetNx("k".into(), "f".into(), "v".into()), - Expect::Keys(&["k"]), - ), - ( - Command::HMGet("k".into(), vec!["f".into()]), - Expect::Keys(&["k"]), - ), - ( - Command::HScan("k".into(), ScanArgs::default()), - Expect::Keys(&["k"]), - ), - ( - Command::LPush("k".into(), vec!["v".into()]), - Expect::Keys(&["k"]), - ), - ( - Command::RPush("k".into(), vec!["v".into()]), - Expect::Keys(&["k"]), - ), - ( - Command::LPushX("k".into(), vec!["v".into()]), - Expect::Keys(&["k"]), - ), - ( - Command::RPushX("k".into(), vec!["v".into()]), - Expect::Keys(&["k"]), - ), - (Command::LPop("k".into(), None), Expect::Keys(&["k"])), - (Command::RPop("k".into(), None), Expect::Keys(&["k"])), - (Command::LRange("k".into(), 0, -1), Expect::Keys(&["k"])), - (Command::LLen("k".into()), Expect::Keys(&["k"])), - (Command::LIndex("k".into(), 0), Expect::Keys(&["k"])), - ( - Command::LSet("k".into(), 0, "v".into()), - Expect::Keys(&["k"]), - ), - ( - Command::LRem("k".into(), 0, "v".into()), - Expect::Keys(&["k"]), - ), - (Command::LTrim("k".into(), 0, -1), Expect::Keys(&["k"])), - ( - Command::SAdd("k".into(), vec!["m".into()]), - Expect::Keys(&["k"]), - ), - (Command::SMembers("k".into()), Expect::Keys(&["k"])), - ( - Command::SRem("k".into(), vec!["m".into()]), - Expect::Keys(&["k"]), - ), - (Command::SCard("k".into()), Expect::Keys(&["k"])), - ( - Command::SIsMember("k".into(), "m".into()), - Expect::Keys(&["k"]), - ), - ( - Command::SMIsMember("k".into(), vec!["m".into()]), - Expect::Keys(&["k"]), - ), - (Command::SInter(vec!["k".into()]), Expect::Keys(&["k"])), - ( - Command::SInterStore("d".into(), vec!["k".into()]), - Expect::Keys(&["k", "d"]), - ), - (Command::SUnion(vec!["k".into()]), Expect::Keys(&["k"])), - ( - Command::SUnionStore("d".into(), vec!["k".into()]), - Expect::Keys(&["k", "d"]), - ), - (Command::SDiff(vec!["k".into()]), Expect::Keys(&["k"])), - ( - Command::SDiffStore("d".into(), vec!["k".into()]), - Expect::Keys(&["k", "d"]), - ), - (Command::SPop("k".into(), None), Expect::Keys(&["k"])), - (Command::SRandMember("k".into(), None), Expect::Keys(&["k"])), - ( - Command::SScan("k".into(), ScanArgs::default()), - Expect::Keys(&["k"]), - ), - ( - Command::SMove("k".into(), "d".into(), "m".into()), - Expect::Keys(&["k", "d"]), - ), - ( - Command::ZAdd("k".into(), ZAddOptions::default(), vec![(1.0, "m".into())]), - Expect::Keys(&["k"]), - ), - ( - Command::ZRange("k".into(), 0, -1, false), - Expect::Keys(&["k"]), - ), - ( - Command::ZRevRange("k".into(), 0, -1, false), - Expect::Keys(&["k"]), - ), - ( - Command::ZRangeByScore("k".into(), "0".into(), "1".into(), false, None), - Expect::Keys(&["k"]), - ), - ( - Command::ZRevRangeByScore("k".into(), "1".into(), "0".into(), false, None), - Expect::Keys(&["k"]), - ), - ( - Command::ZScore("k".into(), "m".into()), - Expect::Keys(&["k"]), - ), - ( - Command::ZMScore("k".into(), vec!["m".into()]), - Expect::Keys(&["k"]), - ), - (Command::ZRank("k".into(), "m".into()), Expect::Keys(&["k"])), - ( - Command::ZRevRank("k".into(), "m".into()), - Expect::Keys(&["k"]), - ), - ( - Command::ZRem("k".into(), vec!["m".into()]), - Expect::Keys(&["k"]), - ), - (Command::ZCard("k".into()), Expect::Keys(&["k"])), - ( - Command::ZIncrBy("k".into(), 1.0, "m".into()), - Expect::Keys(&["k"]), - ), - ( - Command::ZCount("k".into(), "0".into(), "1".into()), - Expect::Keys(&["k"]), - ), - ( - Command::ZScan("k".into(), ScanArgs::default()), - Expect::Keys(&["k"]), - ), - ( - Command::JSet("k".into(), "$".into(), "1".into()), - Expect::Keys(&["k"]), - ), - (Command::JGet("k".into(), None), Expect::Keys(&["k"])), - ( - Command::JMerge("k".into(), "{}".into()), - Expect::Keys(&["k"]), - ), - (Command::RlSet("k".into(), 1, 1), Expect::Keys(&["k"])), - (Command::RlCheck("k".into(), None), Expect::Keys(&["k"])), - (Command::Multi, Expect::KeyLess), - (Command::Exec, Expect::KeyLess), - (Command::Discard, Expect::KeyLess), - (Command::Subscribe(vec!["ch".into()]), Expect::KeyLess), - (Command::Unsubscribe(vec!["ch".into()]), Expect::KeyLess), - (Command::PSubscribe(vec!["ch".into()]), Expect::KeyLess), - (Command::PUnsubscribe(vec!["ch".into()]), Expect::KeyLess), - (Command::Publish("ch".into(), "m".into()), Expect::KeyLess), - (Command::Watch(vec!["k".into()]), Expect::Keys(&["k"])), - (Command::Unwatch(vec!["k".into()]), Expect::Keys(&["k"])), - (Command::Sync(vec![]), Expect::KeyLess), - (Command::QSub("p:*".into()), Expect::KeyLess), - (Command::QUnsub(None), Expect::KeyLess), - (Command::Save, Expect::Admin), - (Command::BgSave, Expect::Admin), - (Command::LastSave, Expect::Admin), - (Command::ReplicaOfNoOne, Expect::Admin), - (Command::Quit, Expect::KeyLess), - (Command::Client(vec!["ID".into()]), Expect::KeyLess), - ( - Command::Config(vec!["GET".into(), "*".into()]), - Expect::Admin, - ), - (Command::CommandQuery(vec![]), Expect::KeyLess), - (Command::Cluster(vec!["INFO".into()]), Expect::KeyLess), - (Command::Module(vec!["LIST".into()]), Expect::KeyLess), - (Command::Memory(vec!["DOCTOR".into()]), Expect::KeyLess), - (Command::MemoryUsage("k".into()), Expect::Keys(&["k"])), - (Command::PubSub(vec!["CHANNELS".into()]), Expect::Admin), - (Command::Unknown("X".into()), Expect::KeyLess), - ] - } - - #[test] - fn every_command_is_classified_for_scope_enforcement() { - for (cmd, expect) in all_commands() { - match (command_scope(&cmd), &expect) { - (CommandScope::KeyLess, Expect::KeyLess) => {} - (CommandScope::Admin, Expect::Admin) => {} - (CommandScope::Keys(got), Expect::Keys(want)) => { - for k in *want { - assert!( - got.contains(&k.to_string()), - "{cmd:?} must scope-check key '{k}', reported {got:?}" - ); - } - } - (got, _) => panic!("{cmd:?} classified as {got:?}, which is not what it touches"), - } - } - } - - #[test] - fn every_key_writing_command_is_classified_as_a_write() { - // `is_write_command` is a `matches!` list, which — unlike a `match` — - // has no exhaustiveness check: a new variant silently defaults to "not - // a write" and is then never replicated, logged to AOF, or broadcast. - // `primary_keys` reports the keys a command *writes*, so anything it - // names must also be classified as a write. This cross-check is what - // makes the missing entry impossible to ship. - for (cmd, _) in all_commands() { - if !primary_keys(&cmd).is_empty() { - assert!( - is_write_command(&cmd), - "{cmd:?} writes keys but is_write_command() says otherwise — \ - it would never reach replicas, the AOF, or live queries" - ); - } - } - } - - #[test] - fn eset_is_a_write_and_reports_its_key() { - let cmd = Command::ESet("presence:1".into(), "on".into()); - assert!(is_write_command(&cmd)); - assert_eq!(primary_keys(&cmd), vec!["presence:1".to_string()]); - assert_eq!(command_name(&cmd), "eset"); - // Scoped connections must not be able to write presence keys outside - // their grant. - assert!(matches!(command_scope(&cmd), CommandScope::Keys(_))); - } - - #[test] - fn eset_replays_to_replicas_as_a_plain_set() { - // A replica has no connection to scope the lifetime to, so it stores an - // ordinary key; the owning server broadcasts the DEL on disconnect. - let frame = broadcast_for( - &Command::ESet("presence:1".into(), "on".into()), - &Value::SimpleString("OK".into()), - ) - .expect("ESET must broadcast"); - let frame = String::from_utf8_lossy(&frame).into_owned(); - assert!(frame.contains("SET"), "{frame}"); - assert!(frame.contains("presence:1")); - assert!( - !frame.contains("ESET"), - "replica should receive SET, not ESET" - ); - } - - #[test] - fn every_command_is_in_the_catalog() { - // The other half of the loop closed in `catalog_names_are_real_commands`: - // every `Command` variant the parser can produce must have a catalog - // row, or `COMMAND DOCS` silently under-reports what the server can do - // and `COMMAND COUNT` lies about how much. - for (cmd, _) in all_commands() { - let name = command_name(&cmd); - if name == "unknown" { - continue; // Not a command — the reply for anything unrecognised. - } - assert!( - catalog::lookup(name).is_some(), - "{name} is a real command with no catalog entry" - ); - } - } - - #[test] - fn command_count_matches_the_catalog() { - let Value::Integer(n) = handle_command_query(&["COUNT".to_string()], 2) else { - panic!("COMMAND COUNT must reply an integer") - }; - assert_eq!(n as usize, catalog::CATALOG.len()); - } - - #[test] - fn command_info_reports_ten_fields_per_entry() { - // Redis 7 returns ten elements. A client that indexes past the sixth - // must find an empty list, not a short array. - let reply = handle_command_query(&["INFO".into(), "get".into()], 2); - let Value::Array(Some(entries)) = reply else { - panic!("expected an array") - }; - let Value::Array(Some(fields)) = &entries[0] else { - panic!("expected an entry array") - }; - assert_eq!(fields.len(), 10); - assert_eq!(fields[0], Value::BulkString(Some(b"get".to_vec()))); - assert_eq!(fields[1], Value::Integer(2)); - assert_eq!(fields[3], Value::Integer(1), "GET's key is at position 1"); - } - - #[test] - fn command_info_nils_unknown_names_in_place() { - let reply = handle_command_query(&["INFO".into(), "get".into(), "nosuchthing".into()], 2); - let Value::Array(Some(entries)) = reply else { - panic!("expected an array") - }; - assert_eq!(entries.len(), 2, "the reply stays aligned with the request"); - assert_eq!(entries[1], Value::Array(None)); - } - - #[test] - fn command_docs_shape_follows_the_protocol() { - // RESP3 gets a map; RESP2 gets the same pairs flattened. - let resp3 = handle_command_query(&["DOCS".into(), "getrange".into()], 3); - assert!(matches!(resp3, Value::Map(_)), "{resp3:?}"); - let resp2 = handle_command_query(&["DOCS".into(), "getrange".into()], 2); - let Value::Array(Some(flat)) = resp2 else { - panic!("RESP2 must flatten the map") - }; - assert_eq!(flat.len(), 2, "one command, one entry"); - assert_eq!(flat[0], Value::BulkString(Some(b"getrange".to_vec()))); - } - - #[test] - fn command_docs_omits_unknown_names() { - let Value::Array(Some(flat)) = - handle_command_query(&["DOCS".into(), "nosuchthing".into()], 2) - else { - panic!("expected an array") - }; - assert!(flat.is_empty(), "an unknown name has no entry to key"); - } - - #[test] - fn command_rejects_unknown_subcommands() { - let reply = handle_command_query(&["GETKEYS".into(), "get".into(), "k".into()], 2); - assert!( - matches!(&reply, Value::Error(e) if e.contains("Unknown subcommand")), - "{reply:?}" - ); - } - - #[test] - fn client_setinfo_is_recorded_and_visible() { - let mut meta = ClientMeta::new(42, "127.0.0.1:1".into(), "127.0.0.1:6379".into()); - assert_eq!( - handle_client_command( - &["SETINFO".into(), "LIB-NAME".into(), "node-redis".into()], - &mut meta - ), - Value::SimpleString("OK".into()) - ); - assert_eq!( - handle_client_command( - &["SETINFO".into(), "LIB-VER".into(), "6.2.0".into()], - &mut meta - ), - Value::SimpleString("OK".into()) - ); - let Value::BulkString(Some(line)) = handle_client_command(&["INFO".into()], &mut meta) - else { - panic!("CLIENT INFO must reply a bulk string") - }; - let line = String::from_utf8(line).unwrap(); - assert!(line.contains("lib-name=node-redis"), "{line}"); - assert!(line.contains("lib-ver=6.2.0"), "{line}"); - assert!(line.contains("id=42"), "{line}"); - } - - #[test] - fn client_setinfo_rejects_unknown_attributes() { - let mut meta = ClientMeta::new(1, String::new(), String::new()); - let reply = handle_client_command( - &["SETINFO".into(), "LIB-COLOUR".into(), "blue".into()], - &mut meta, - ); - assert!( - matches!(&reply, Value::Error(e) if e.contains("Unrecognized")), - "{reply:?}" - ); - } - - #[test] - fn client_setname_round_trips_and_rejects_spaces() { - let mut meta = ClientMeta::new(1, String::new(), String::new()); - assert_eq!( - handle_client_command(&["GETNAME".into()], &mut meta), - Value::BulkString(None), - "an unnamed connection reports nil, not an empty string" - ); - handle_client_command(&["SETNAME".into(), "worker-3".into()], &mut meta); - assert_eq!( - handle_client_command(&["GETNAME".into()], &mut meta), - Value::BulkString(Some(b"worker-3".to_vec())) - ); - // A space would break the key=value line CLIENT LIST emits. - let reply = handle_client_command(&["SETNAME".into(), "two words".into()], &mut meta); - assert!(matches!(reply, Value::Error(_)), "{reply:?}"); - } - - #[test] - fn client_declines_what_it_cannot_do() { - // KILL must not answer +OK: a caller would believe a connection had - // been closed when it is still open. - let mut meta = ClientMeta::new(1, String::new(), String::new()); - for args in [ - vec!["KILL".to_string(), "id".into(), "3".into()], - vec!["NO-EVICT".to_string(), "on".into()], - vec!["UNPAUSE".to_string()], - ] { - let reply = handle_client_command(&args, &mut meta); - assert!( - matches!(&reply, Value::Error(e) if e.contains("Unknown subcommand")), - "{args:?} -> {reply:?}" - ); - } - } - - #[test] - fn config_get_reports_values_in_force() { - let store = KeyValueStore::new(); - let facts = test_facts(); - let Value::Array(Some(flat)) = - handle_config_command(&["GET".into(), "maxmemory-policy".into()], &facts, &store) - else { - panic!("CONFIG GET must reply an array") - }; - assert_eq!(flat.len(), 2); - assert_eq!( - flat[0], - Value::BulkString(Some(b"maxmemory-policy".to_vec())) - ); - assert_eq!( - flat[1], - Value::BulkString(Some( - eviction_policy_name(store.eviction_policy()) - .as_bytes() - .to_vec() - )), - "the reported policy must be the one actually in force" - ); - } - - #[test] - fn config_get_matches_globs_and_multiple_names() { - let store = KeyValueStore::new(); - let facts = test_facts(); - let Value::Array(Some(flat)) = - handle_config_command(&["GET".into(), "maxmemory*".into()], &facts, &store) - else { - panic!("expected an array") - }; - // maxmemory and maxmemory-policy both match; the reply is flat pairs. - assert_eq!(flat.len(), 4, "{flat:?}"); - - let Value::Array(Some(none)) = - handle_config_command(&["GET".into(), "nosuchparam".into()], &facts, &store) - else { - panic!("expected an array") - }; - assert!( - none.is_empty(), - "an unmatched name yields no pair, not an error" - ); - } - - #[test] - fn config_get_masks_the_password() { - let store = KeyValueStore::new(); - let mut facts = test_facts(); - facts.auth_enabled = true; - let Value::Array(Some(flat)) = - handle_config_command(&["GET".into(), "requirepass".into()], &facts, &store) - else { - panic!("expected an array") - }; - assert_eq!( - flat[1], - Value::BulkString(Some(b"*".to_vec())), - "the password itself must never leave the process" - ); - } - - #[test] - fn config_set_refuses_rather_than_pretending() { - let store = KeyValueStore::new(); - let facts = test_facts(); - let reply = handle_config_command( - &["SET".into(), "maxmemory".into(), "100mb".into()], - &facts, - &store, - ); - // Nothing in the running server can change, so +OK would be a lie the - // operator only discovers when the limit fails to apply. - assert!( - matches!(&reply, Value::Error(e) if e.contains("configured at startup")), - "{reply:?}" - ); - } - - #[test] - fn every_command_has_a_metrics_label() { - for (cmd, _) in all_commands() { - let name = command_name(&cmd); - assert!(!name.is_empty(), "{cmd:?} has an empty metrics label"); - assert_eq!( - name, - name.to_lowercase(), - "{cmd:?} label '{name}' must be lowercase for Prometheus" - ); - } - } - - #[test] - fn primary_keys_reports_writes_only() { - // `primary_keys` answers "what did this command *write*", for - // replication and push targeting — it is deliberately NOT the - // authorization function (that is `command_scope`). Reads report - // nothing because there is no mutation to broadcast. - for cmd in [ - Command::Get("k".into()), - Command::Exists(vec!["k".into()]), - Command::Ttl("k".into()), - Command::LRange("k".into(), 0, -1), - Command::SMembers("k".into()), - Command::HGetAll("k".into()), - ] { - assert!( - primary_keys(&cmd).is_empty(), - "{cmd:?} is a read and must not be broadcast as a mutation" - ); - } - - // Writes must report every key they touch, or a replica or subscribed - // browser silently misses the change. - let writes: Vec<(Command, &[&str])> = vec![ - ( - Command::Set("k".into(), "v".into(), SetOptions::default()), - &["k"], - ), - (Command::Del(vec!["a".into(), "b".into()]), &["a", "b"]), - (Command::Incr("k".into()), &["k"]), - (Command::Rename("src".into(), "dst".into()), &["src", "dst"]), - ( - Command::SMove("src".into(), "dst".into(), "m".into()), - &["src", "dst"], - ), - ( - Command::MSet(vec![("a".into(), "1".into()), ("b".into(), "2".into())]), - &["a", "b"], - ), - ( - Command::SInterStore("dst".into(), vec!["a".into()]), - &["dst"], - ), - (Command::LPush("k".into(), vec!["v".into()]), &["k"]), - ( - Command::HSet("k".into(), vec![("f".into(), "v".into())]), - &["k"], - ), - (Command::JMerge("k".into(), "{}".into()), &["k"]), - ]; - for (cmd, want) in writes { - let got = primary_keys(&cmd); - for k in want { - assert!( - got.contains(&k.to_string()), - "{cmd:?}: primary_keys missed written key '{k}', got {got:?}" - ); - } - } - } - - // ── Pub/Sub pattern matching ────────────────────────────────────────────── - - #[test] - fn psubscribe_pattern_matching_is_not_exponential() { - // PSUBSCRIBE patterns are attacker-controlled, and every PUBLISH is - // matched against every registered pattern. This file previously used a - // recursive matcher that backtracked exponentially: a 10-wildcard - // pattern against a 36-character channel took ~7 s, so one subscriber - // could stall pub/sub for everyone. It now shares core-engine's DP - // matcher (verified equivalent). This test fails loudly if that - // regresses. - let pattern = "*a*a*a*a*a*a*a*a*a*a*b"; - let channel = "a".repeat(200); - let start = std::time::Instant::now(); - assert!(!core_engine::store::glob_match(pattern, &channel)); - assert!( - start.elapsed() < std::time::Duration::from_millis(500), - "pattern matching took {:?} — exponential backtracking is back", - start.elapsed() - ); - } - - #[test] - fn pubsub_patterns_match_the_expected_channels() { - for (pat, ch, want) in [ - ("news.*", "news.tech", true), - ("news.*", "news.", true), - ("news.*", "sports.tech", false), - ("*", "anything", true), - ("user.?", "user.1", true), - ("user.?", "user.42", false), - ] { - assert_eq!( - core_engine::store::glob_match(pat, ch), - want, - "pattern {pat:?} vs channel {ch:?}" - ); - } - } - - // ── Introspection: PUBSUB / CLUSTER / MODULE / MEMORY ───────────────────── - - /// A hub with `channels` subscribed and `patterns` psubscribed. The senders - /// are kept alive by the returned vector — dropping them would close the - /// receivers and make the hub look empty. - fn hub_with( - channels: &[(u64, &str)], - patterns: &[(u64, &str)], - ) -> (PubSubHub, Vec>) { - let mut hub = PubSubHub::new(); - let mut keepalive = Vec::new(); - for (id, ch) in channels { - let (tx, rx) = mpsc::unbounded_channel(); - hub.subscribe(*id, ch, tx); - keepalive.push(rx); - } - for (id, pat) in patterns { - let (tx, rx) = mpsc::unbounded_channel(); - hub.psubscribe(*id, pat, tx); - keepalive.push(rx); - } - (hub, keepalive) - } - - fn bulk_strings(v: &Value) -> Vec { - match v { - Value::Array(Some(items)) => items - .iter() - .map(|i| match i { - Value::BulkString(Some(b)) => String::from_utf8_lossy(b).into_owned(), - other => panic!("expected a bulk string, got {other:?}"), - }) - .collect(), - other => panic!("expected an array, got {other:?}"), - } - } - - #[test] - fn pubsub_channels_lists_only_channels_with_subscribers() { - let (hub, _keep) = hub_with(&[(1, "news"), (2, "news"), (3, "sports")], &[(4, "news.*")]); - - let mut all = bulk_strings(&handle_pubsub_command(&["CHANNELS".into()], &hub)); - all.sort(); - assert_eq!(all, vec!["news".to_string(), "sports".to_string()]); - - // A pattern subscriber is not a channel. Redis reports `news.*` under - // NUMPAT and never under CHANNELS, because nobody is subscribed to a - // channel by that name. - assert!(!all.contains(&"news.*".to_string())); - - let filtered = bulk_strings(&handle_pubsub_command( - &["CHANNELS".into(), "spo*".into()], - &hub, - )); - assert_eq!(filtered, vec!["sports".to_string()]); - } - - #[test] - fn pubsub_numsub_counts_per_channel_and_keeps_the_caller_s_order() { - let (hub, _keep) = hub_with(&[(1, "news"), (2, "news"), (3, "sports")], &[(4, "news.*")]); - - let reply = handle_pubsub_command( - &[ - "NUMSUB".into(), - "sports".into(), - "news".into(), - "nobody-here".into(), - ], - &hub, - ); - assert_eq!( - reply, - Value::Array(Some(vec![ - Value::BulkString(Some(b"sports".to_vec())), - Value::Integer(1), - Value::BulkString(Some(b"news".to_vec())), - // Two subscribers, and the `news.*` pattern subscriber is not - // one of them: NUMPAT's job, counted here would be double. - Value::Integer(2), - Value::BulkString(Some(b"nobody-here".to_vec())), - // Present with a zero rather than omitted, so a caller can read - // the reply by position against the channels it asked about. - Value::Integer(0), - ])) - ); - - // No channels named is a legal call and an empty reply, not an error. - assert_eq!( - handle_pubsub_command(&["NUMSUB".into()], &hub), - Value::Array(Some(vec![])) - ); - } - - #[test] - fn pubsub_numpat_counts_distinct_patterns_not_subscribers() { - let (hub, _keep) = hub_with(&[], &[(1, "news.*"), (2, "news.*"), (3, "sports.*")]); - assert_eq!( - handle_pubsub_command(&["NUMPAT".into()], &hub), - Value::Integer(2), - "two clients on one pattern are one pattern" - ); - } - - #[test] - fn pubsub_channels_forgets_a_channel_once_its_last_subscriber_leaves() { - let (mut hub, _keep) = hub_with(&[(1, "news"), (2, "news")], &[]); - hub.unsubscribe(1, "news"); - assert_eq!( - bulk_strings(&handle_pubsub_command(&["CHANNELS".into()], &hub)), - vec!["news".to_string()] - ); - hub.unsubscribe(2, "news"); - assert!( - bulk_strings(&handle_pubsub_command(&["CHANNELS".into()], &hub)).is_empty(), - "an abandoned channel is not an active channel" - ); - } - - #[test] - fn pubsub_refuses_the_sharded_subcommands() { - let (hub, _keep) = hub_with(&[], &[]); - // A standalone redis-server answers these with an empty array, and this - // is the one place Recached deliberately does not match it: there, the - // empty array sits next to a working SSUBSCRIBE. Here there is none, so - // "no shard channels are subscribed" would invite a call that fails. - for sub in ["SHARDCHANNELS", "SHARDNUMSUB"] { - assert!( - matches!( - handle_pubsub_command(&[sub.to_string()], &hub), - Value::Error(_) - ), - "{sub} should be refused" - ); - } - } - - #[test] - fn cluster_is_refused_the_way_a_standalone_redis_refuses_it() { - // Verified against redis-server 7.2.5: a server not started in cluster - // mode rejects the whole CLUSTER container with this sentence. It does - // *not* answer INFO with cluster_enabled:0 — that lives in `INFO`. - for sub in ["INFO", "NODES", "SLOTS", "MYID", "SHARDS"] { - assert_eq!( - handle_cluster_command(&[sub.to_string()]), - Value::Error("ERR This instance has cluster support disabled".to_string()), - "CLUSTER {sub}" - ); - } - } - - #[test] - fn info_publishes_the_cluster_flag_that_cluster_info_cannot() { - let store = KeyValueStore::new(); - let body = render_info( - &["cluster".to_string()], - server_facts(), - &store, - sampled_keyspace(&store), - false, - ReplInfo::default(), - 0, - 0, - 0, - ); - assert!(body.contains("# Cluster\r\n"), "section header: {body:?}"); - assert!(body.contains("cluster_enabled:0"), "{body:?}"); - - // And it is in the default set, so a client that sends a bare INFO — - // which is what every cluster-aware client actually sends — sees it. - let default = render_info( - &[], - server_facts(), - &store, - sampled_keyspace(&store), - false, - ReplInfo::default(), - 0, - 0, - 0, - ); - assert!(default.contains("cluster_enabled:0"), "{default:?}"); - } - - #[test] - fn module_list_is_empty_and_loading_is_refused() { - assert_eq!( - handle_module_command(&["LIST".to_string()]), - Value::Array(Some(vec![])), - "no modules is an answer, not an error" - ); - for sub in ["LOAD", "LOADEX", "UNLOAD"] { - assert!( - matches!( - handle_module_command(&[sub.to_string(), "/tmp/x.so".to_string()]), - Value::Error(_) - ), - "MODULE {sub} should be refused rather than answered +OK" - ); - } - } - - #[test] - fn memory_allocator_subcommands_are_refused_with_a_reason() { - for sub in ["DOCTOR", "STATS", "PURGE", "MALLOC-STATS"] { - let Value::Error(msg) = handle_memory_command(&[sub.to_string()]) else { - panic!("MEMORY {sub} should be refused"); - }; - assert!( - msg.contains("MEMORY USAGE"), - "the refusal should name what does work: {msg}" - ); - } - assert!(matches!( - handle_memory_command(&["HELP".to_string()]), - Value::Array(Some(_)) - )); - } - - // ── Wire encoding ───────────────────────────────────────────────────────── - - #[test] - fn pubsub_message_encodes_as_a_resp3_push_frame() { - let bytes = encode_pubsub_msg( - PubSubMsg::Message { - channel: "news".into(), - message: "hello".into(), - }, - 3, - ); - let text = String::from_utf8_lossy(&bytes); - assert!( - text.starts_with('>'), - "must be a RESP3 Push frame: {text:?}" - ); - assert!(text.contains("message")); - assert!(text.contains("news")); - assert!(text.contains("hello")); - } - - #[test] - fn pubsub_message_encodes_as_an_array_for_resp2() { - // RESP2 has no push type. Sending `>` to a RESP2 client — which is - // every client that has not sent HELLO 3 — is unparseable, so a - // subscribed connection would break outright. - let bytes = encode_pubsub_msg( - PubSubMsg::Message { - channel: "news".into(), - message: "hello".into(), - }, - 2, - ); - let text = String::from_utf8_lossy(&bytes); - assert!( - text.starts_with("*3\r\n"), - "RESP2 delivery must be a 3-element array: {text:?}" - ); - assert!(!text.contains('>'), "no push frame on RESP2: {text:?}"); - } - - #[test] - fn pattern_message_carries_the_matching_pattern() { - // A pmessage must name the pattern that matched, or a client - // subscribed to several patterns cannot tell them apart. - for protover in [2u8, 3u8] { - let bytes = encode_pubsub_msg( - PubSubMsg::PMessage { - pattern: "news.*".into(), - channel: "news.tech".into(), - message: "hi".into(), - }, - protover, - ); - let text = String::from_utf8_lossy(&bytes); - assert!(text.contains("pmessage"), "protover {protover}"); - assert!(text.contains("news.*"), "protover {protover}"); - assert!(text.contains("news.tech"), "protover {protover}"); - } - } - - // ── HELLO / protocol negotiation ───────────────────────────────────────── - - #[test] - fn hello_defaults_to_the_connections_current_version() { - // Bare HELLO reports, it does not change. A client using it purely to - // read server info must not be silently switched to another protocol. - let mut protover = 2u8; - let bytes = process_hello(None, &mut protover, true, false); - assert_eq!(protover, 2, "bare HELLO must not change the version"); - let text = String::from_utf8_lossy(&bytes); - assert!( - text.starts_with('*'), - "RESP2 reply must be an array: {text:?}" - ); - assert!(text.contains("recached")); - assert!(text.contains(":2\r\n"), "proto must report 2: {text:?}"); - } - - #[test] - fn hello_3_upgrades_and_replies_with_a_map() { - let mut protover = 2u8; - let bytes = process_hello(Some("3"), &mut protover, true, false); - assert_eq!(protover, 3); - let text = String::from_utf8_lossy(&bytes); - assert!(text.starts_with("%6\r\n"), "must be a 6-pair map: {text:?}"); - assert!(text.contains(":3\r\n"), "proto must report 3: {text:?}"); - } - - #[test] - fn hello_3_then_2_downgrades_again() { - let mut protover = 2u8; - process_hello(Some("3"), &mut protover, true, false); - assert_eq!(protover, 3); - let bytes = process_hello(Some("2"), &mut protover, true, false); - assert_eq!(protover, 2, "HELLO 2 must downgrade"); - assert!(String::from_utf8_lossy(&bytes).starts_with('*')); - } - - #[test] - fn hello_rejects_unsupported_versions_without_changing_protocol() { - // A client probing for a version the server does not speak must get a - // clean NOPROTO and stay on what it had — not be left in a half-state. - for bad in ["4", "1", "0", "abc", "", "255", "-1", "3.0"] { - let mut protover = 2u8; - let bytes = process_hello(Some(bad), &mut protover, true, false); - let text = String::from_utf8_lossy(&bytes); - assert!( - text.starts_with("-NOPROTO"), - "HELLO {bad:?} must be refused: {text:?}" - ); - assert_eq!(protover, 2, "HELLO {bad:?} must not change the version"); - } - } - - #[test] - fn hello_does_not_leak_server_details_before_auth() { - let mut protover = 2u8; - let bytes = process_hello(Some("3"), &mut protover, false, false); - let text = String::from_utf8_lossy(&bytes); - assert!(text.starts_with("-NOAUTH"), "{text:?}"); - assert!( - !text.contains("recached") && !text.contains(env!("CARGO_PKG_VERSION")), - "unauthenticated HELLO must not fingerprint the server: {text:?}" - ); - } - - #[test] - fn hello_reports_replica_role() { - let mut protover = 3u8; - let primary = - String::from_utf8_lossy(&process_hello(None, &mut protover, true, false)).into_owned(); - let replica = - String::from_utf8_lossy(&process_hello(None, &mut protover, true, true)).into_owned(); - assert!(primary.contains("master"), "{primary:?}"); - assert!(replica.contains("replica"), "{replica:?}"); - } - - // ── INFO ────────────────────────────────────────────────────────────────── - - fn test_facts() -> ServerFacts { - ServerFacts { - start: SystemTime::now() - std::time::Duration::from_secs(90_000), - run_id: "a".repeat(40), - tcp_port: 6379, - ws_port: 6380, - max_connections: 512, - tls_enabled: true, - auth_enabled: true, - aof_enabled: true, - } - } - - /// Render `sections` against `store`, walking it for the keyspace numbers. - /// - /// Tests pass the sample explicitly rather than going through the shared - /// 5s cache — `render_info` is pure so that concurrent tests cannot - /// observe each other's keyspace through a process-global. - fn info_for(sections: &[&str], store: &KeyValueStore) -> String { - render_info( - §ions.iter().map(|s| s.to_string()).collect::>(), - &test_facts(), - store, - store.keyspace_sample(), - false, - ReplInfo::default(), - 1_700_000_000, - 0, - 0, - ) - } - - /// Parse an INFO payload into (section, field) → value, enforcing the shape - /// clients rely on: CRLF endings, `# Section` headers, `field:value` lines. - fn parse_info(payload: &str) -> HashMap<(String, String), String> { - assert!( - !payload.contains('\n') || payload.contains("\r\n"), - "INFO must use CRLF line endings" - ); - let mut out = HashMap::new(); - let mut section = String::new(); - for line in payload.split("\r\n") { - if line.is_empty() { - continue; - } - if let Some(name) = line.strip_prefix("# ") { - section = name.to_lowercase(); - continue; - } - let (k, v) = line - .split_once(':') - .unwrap_or_else(|| panic!("malformed INFO line: {line:?}")); - out.insert((section.clone(), k.to_string()), v.to_string()); - } - out - } - - #[test] - fn info_default_emits_every_default_section() { - let store = KeyValueStore::new(); - let payload = info_for(&[], &store); - for section in DEFAULT_INFO_SECTIONS { - let header = format!("# {}{}\r\n", section[..1].to_uppercase(), §ion[1..]); - assert!( - payload.contains(&header), - "missing section header {header:?} in {payload:?}" - ); - } - } - - #[test] - fn info_uses_crlf_and_blank_line_separated_sections() { - let store = KeyValueStore::new(); - let payload = info_for(&["server", "clients"], &store); - assert!(payload.starts_with("# Server\r\n"), "{payload:?}"); - // A blank line must close each section, or parsers merge them. - assert!(payload.contains("\r\n\r\n# Clients\r\n"), "{payload:?}"); - assert!(payload.ends_with("\r\n\r\n"), "{payload:?}"); - assert!(!payload.contains('\n') || !payload.replace("\r\n", "").contains('\n')); - } - - #[test] - fn info_server_section_reports_compat_version_separately_from_ours() { - let store = KeyValueStore::new(); - let f = parse_info(&info_for(&["server"], &store)); - // Clients feature-gate on redis_version, so it must be a Redis version, - // never Recached's own — that is the entire point of the split. - assert_eq!(f[&("server".into(), "redis_version".into())], "6.2.0"); - assert_eq!( - f[&("server".into(), "recached_version".into())], - env!("CARGO_PKG_VERSION") - ); - assert_eq!(f[&("server".into(), "redis_mode".into())], "standalone"); - assert_eq!(f[&("server".into(), "tcp_port".into())], "6379"); - assert_eq!(f[&("server".into(), "recached_ws_port".into())], "6380"); - assert_eq!(f[&("server".into(), "run_id".into())].len(), 40); - // 90_000s of uptime is one day and change. - assert_eq!(f[&("server".into(), "uptime_in_days".into())], "1"); - assert!( - f[&("server".into(), "uptime_in_seconds".into())] - .parse::() - .unwrap() - >= 90_000 - ); - } - - #[test] - fn info_memory_section_reports_limits_and_policy() { - let store = - KeyValueStore::with_config(Some(50), Some(1024 * 1024), EvictionPolicy::AllKeysLru); - let f = parse_info(&info_for(&["memory"], &store)); - assert_eq!(f[&("memory".into(), "maxmemory".into())], "1048576"); - assert_eq!(f[&("memory".into(), "maxmemory_human".into())], "1.00M"); - assert_eq!( - f[&("memory".into(), "maxmemory_policy".into())], - "allkeys-lru" - ); - assert_eq!(f[&("memory".into(), "recached_max_keys".into())], "50"); - } - - #[test] - fn info_memory_reports_zero_maxmemory_when_unbounded() { - // Redis reports 0 for "no limit"; None must not leak as a debug string. - let f = parse_info(&info_for(&["memory"], &KeyValueStore::new())); - assert_eq!(f[&("memory".into(), "maxmemory".into())], "0"); - assert_eq!( - f[&("memory".into(), "maxmemory_policy".into())], - "noeviction" - ); - } - - #[test] - fn info_persistence_always_reports_loading_zero() { - // A client's ready-check gates on this field; the snapshot is loaded - // before any listener binds, so a reachable server is never loading. - let f = parse_info(&info_for(&["persistence"], &KeyValueStore::new())); - assert_eq!(f[&("persistence".into(), "loading".into())], "0"); - assert_eq!( - f[&("persistence".into(), "rdb_last_save_time".into())], - "1700000000" - ); - assert_eq!(f[&("persistence".into(), "aof_enabled".into())], "1"); - } - - #[test] - fn info_replication_reports_both_redis_and_recached_spellings() { - let store = KeyValueStore::new(); - let repl = ReplInfo { - connected: 2, - queue_depth: 7, - lag_frames: 3, - }; - let primary = parse_info(&render_info( - &[], - &test_facts(), - &store, - store.keyspace_sample(), - false, - repl, - 0, - 0, - 0, - )); - assert_eq!(primary[&("replication".into(), "role".into())], "master"); - // Tooling greps for `connected_slaves`; the modern alias ships too. - assert_eq!( - primary[&("replication".into(), "connected_slaves".into())], - "2" - ); - assert_eq!( - primary[&("replication".into(), "connected_replicas".into())], - "2" - ); - assert_eq!( - primary[&( - "replication".into(), - "recached_replication_lag_frames".into() - )], - "3" - ); - - let replica = parse_info(&render_info( - &[], - &test_facts(), - &store, - store.keyspace_sample(), - true, - ReplInfo::default(), - 0, - 0, - 0, - )); - // Redis still spells a replica `slave` in INFO, and clients match on it. - assert_eq!(replica[&("replication".into(), "role".into())], "slave"); - } - - #[test] - fn info_keyspace_omits_the_db_line_when_empty_and_counts_ttls_when_not() { - let store = KeyValueStore::new(); - assert!( - !info_for(&["keyspace"], &store).contains("db0:"), - "an empty keyspace must not report a db0 line" - ); - - store.execute(Command::Set( - "a".into(), - b"v".to_vec(), - SetOptions::default(), - )); - store.execute(Command::Set( - "b".into(), - b"v".to_vec(), - SetOptions::default(), - )); - store.execute(Command::Expire("b".into(), 60)); - let payload = info_for(&["keyspace"], &store); - assert!( - payload.contains("db0:keys=2,expires=1,avg_ttl=0"), - "{payload:?}" - ); - } - - #[test] - fn sampled_keyspace_falls_back_to_a_live_walk_before_the_sampler_runs() { - // First INFO of a process arrives before the 5s sampler has ever run, - // and must not report an empty keyspace. - let store = KeyValueStore::new(); - store.execute(Command::Set( - "k".into(), - b"v".to_vec(), - SetOptions::default(), - )); - SAMPLED_KEYS.store(u64::MAX, Ordering::Relaxed); - assert_eq!(sampled_keyspace(&store).keys, 1); - } - - #[test] - fn info_unknown_section_yields_nothing() { - // Redis answers an unknown section with an empty payload, not an error. - assert_eq!(info_for(&["nosuchsection"], &KeyValueStore::new()), ""); - } - - #[test] - fn info_all_and_everything_expand_to_the_default_sections() { - let store = KeyValueStore::new(); - let default = info_for(&[], &store); - for alias in ["all", "everything", "default"] { - assert_eq!( - info_for(&[alias], &store).lines().count(), - default.lines().count(), - "INFO {alias} must cover the default sections" - ); - } - } - - #[test] - fn info_honours_section_selection_and_order() { - let payload = info_for(&["clients", "server"], &KeyValueStore::new()); - assert!(payload.starts_with("# Clients\r\n"), "{payload:?}"); - assert!(payload.contains("# Server\r\n"), "{payload:?}"); - assert!(!payload.contains("# Memory"), "{payload:?}"); - } - - #[test] - fn human_bytes_matches_redis_formatting() { - assert_eq!(human_bytes(0), "0B"); - assert_eq!(human_bytes(512), "512B"); - assert_eq!(human_bytes(1024), "1.00K"); - assert_eq!(human_bytes(1024 * 1024), "1.00M"); - assert_eq!(human_bytes(3 * 1024 * 1024 * 1024), "3.00G"); - } - - #[test] - fn run_ids_are_forty_hex_chars_and_differ_per_process() { - let a = generate_run_id(); - assert_eq!(a.len(), 40); - assert!(a.chars().all(|c| c.is_ascii_hexdigit()), "{a}"); - assert_ne!(a, generate_run_id()); - } - - #[test] - fn info_is_administrative_scope() { - // Scoped WebSocket connections must not be able to read server-wide - // state, so INFO has to classify as Admin, not KeyLess. - assert!(matches!( - command_scope(&Command::Info(vec![])), - CommandScope::Admin - )); - } - - #[test] - fn info_is_not_a_write_command() { - assert!(!is_write_command(&Command::Info(vec![]))); - } - - #[tokio::test] - async fn info_over_tcp_returns_a_parseable_bulk_string() { - let srv = spawn_server().await; - let mut c = RespClient::connect(srv.tcp_addr).await; - - let Value::BulkString(Some(bytes)) = c.cmd(&["INFO"]).await else { - panic!("INFO must reply with a bulk string"); - }; - let payload = String::from_utf8(bytes).unwrap(); - let f = parse_info(&payload); - assert_eq!(f[&("server".into(), "redis_version".into())], "6.2.0"); - assert_eq!(f[&("replication".into(), "role".into())], "master"); - assert!(f.contains_key(&("stats".into(), "total_commands_processed".into()))); - } - - #[tokio::test] - async fn info_section_argument_is_honoured_over_the_wire() { - let srv = spawn_server().await; - let mut c = RespClient::connect(srv.tcp_addr).await; - - let Value::BulkString(Some(bytes)) = c.cmd(&["INFO", "server"]).await else { - panic!("INFO must reply with a bulk string"); - }; - let payload = String::from_utf8(bytes).unwrap(); - assert!(payload.starts_with("# Server\r\n"), "{payload:?}"); - assert!(!payload.contains("# Memory"), "{payload:?}"); - } - - #[tokio::test] - async fn info_reflects_live_server_state() { - let srv = spawn_server().await; - let mut c = RespClient::connect(srv.tcp_addr).await; - - c.cmd(&["SET", "k", "v"]).await; - c.cmd(&["GET", "k"]).await; // hit - c.cmd(&["GET", "missing"]).await; // miss - - let Value::BulkString(Some(bytes)) = c.cmd(&["INFO", "stats", "persistence"]).await else { - panic!("INFO must reply with a bulk string"); - }; - let f = parse_info(&String::from_utf8(bytes).unwrap()); - assert!( - f[&("stats".into(), "keyspace_hits".into())] - .parse::() - .unwrap() - >= 1 - ); - assert!( - f[&("stats".into(), "keyspace_misses".into())] - .parse::() - .unwrap() - >= 1 - ); - // The SET must show up as an unsaved change. - assert!( - f[&("persistence".into(), "rdb_changes_since_last_save".into())] - .parse::() - .unwrap() - >= 1 - ); - } - - #[tokio::test] - async fn info_requires_authentication() { - let srv = spawn_server_cfg(Some("hunter2"), None, false).await; - let mut c = RespClient::connect(srv.tcp_addr).await; - - // INFO leaks deployment details, so it must sit behind AUTH like every - // other non-handshake command. - match c.cmd(&["INFO"]).await { - Value::Error(e) => assert!(e.starts_with("NOAUTH"), "{e}"), - other => panic!("unauthenticated INFO must be refused, got {other:?}"), - } - - assert_eq!(c.cmd(&["AUTH", "hunter2"]).await, ok()); - assert!(matches!(c.cmd(&["INFO"]).await, Value::BulkString(Some(_)))); - } - - #[tokio::test] - async fn info_on_a_replica_reports_the_slave_role() { - let srv = spawn_server_cfg(None, None, true).await; - let mut c = RespClient::connect(srv.tcp_addr).await; - - let Value::BulkString(Some(bytes)) = c.cmd(&["INFO", "replication"]).await else { - panic!("INFO must reply with a bulk string"); - }; - let f = parse_info(&String::from_utf8(bytes).unwrap()); - assert_eq!(f[&("replication".into(), "role".into())], "slave"); - } - - #[test] - fn subscribe_ack_reports_the_running_subscription_count() { - let bytes = resp_subscribe_ack("subscribe", "news", 3); - let text = String::from_utf8_lossy(&bytes); - assert!(text.contains("subscribe")); - assert!(text.contains("news")); - assert!( - text.contains(":3"), - "count must be a RESP integer: {text:?}" - ); - } - - // ── Score formatting ────────────────────────────────────────────────────── - - #[test] - fn scores_format_without_trailing_decimals() { - // Redis returns "1" not "1.0" — clients parse these as integers. - assert_eq!(format_f64_score(1.0), "1"); - assert_eq!(format_f64_score(-5.0), "-5"); - assert_eq!(format_f64_score(0.0), "0"); - assert_eq!(format_f64_score(1.5), "1.5"); - assert_eq!(format_f64_score(-0.25), "-0.25"); - } - - #[test] - fn scores_format_infinities_as_redis_does() { - assert_eq!(format_f64_score(f64::INFINITY), "inf"); - assert_eq!(format_f64_score(f64::NEG_INFINITY), "-inf"); - } - - #[test] - fn very_large_scores_do_not_lose_their_exponent() { - // Past 1e15 the integer shortcut is skipped, because casting to i64 - // would silently truncate. - let big = 1e16_f64; - let s = format_f64_score(big); - assert!( - s.contains('e') || s.len() > 15, - "unexpected formatting: {s}" - ); - } - - // ── Save conditions ─────────────────────────────────────────────────────── - - #[test] - fn save_conditions_parse_as_seconds_colon_changes() { - let c = parse_save_conditions("900:1,300:10,60:10000"); - assert_eq!(c.len(), 3); - assert_eq!(c[0].secs, 900); - assert_eq!(c[0].changes, 1); - assert_eq!(c[2].secs, 60); - assert_eq!(c[2].changes, 10000); - } - - #[test] - fn save_conditions_tolerate_whitespace() { - let c = parse_save_conditions(" 900 : 1 , 300 : 10 "); - assert_eq!(c.len(), 2); - assert_eq!(c[0].secs, 900); - assert_eq!(c[1].changes, 10); - } - - #[test] - fn malformed_save_conditions_are_skipped_not_fatal() { - // A bad pair is dropped so one typo cannot disable autosave entirely — - // but a wholly invalid string yields no conditions, which the caller - // treats as "autosave off". - let c = parse_save_conditions("900:1,garbage,300:10"); - assert_eq!(c.len(), 2, "valid pairs survive a bad one"); - assert!(parse_save_conditions("").is_empty()); - assert!(parse_save_conditions("nonsense").is_empty()); - assert!( - parse_save_conditions("900").is_empty(), - "missing ':changes'" - ); - } - - // ── TLS configuration ───────────────────────────────────────────────────── - - #[test] - fn tls_requires_both_cert_and_key() { - assert_eq!( - resolve_tls_paths(None, None).unwrap(), - None, - "neither set → plaintext" - ); - assert_eq!( - resolve_tls_paths(Some("c.pem".into()), Some("k.pem".into())).unwrap(), - Some(("c.pem".to_string(), "k.pem".to_string())) - ); - } - - #[test] - fn tls_half_configured_is_refused_not_downgraded() { - // The dangerous case: an operator sets the cert, mistypes the key - // variable, and the server used to serve plaintext on both ports while - // reporting itself healthy. Traffic believed encrypted was not. - let cert_only = resolve_tls_paths(Some("c.pem".into()), None).unwrap_err(); - assert!(cert_only.contains("RECACHED_TLS_KEY"), "got {cert_only}"); - assert!( - cert_only.contains("plaintext"), - "must explain the risk: {cert_only}" - ); - - let key_only = resolve_tls_paths(None, Some("k.pem".into())).unwrap_err(); - assert!(key_only.contains("RECACHED_TLS_CERT"), "got {key_only}"); - } - - // ── IP allowlist ────────────────────────────────────────────────────────── - - #[test] - fn allow_ips_parses_exact_addresses() { - let ips = parse_allow_ips("10.0.1.5, 10.0.1.6").unwrap(); - assert_eq!(ips.len(), 2); - assert!(ips.contains(&IpAddr::from_str("10.0.1.5").unwrap())); - // IPv6 literals are accepted too. - let v6 = parse_allow_ips("::1").unwrap(); - assert_eq!(v6, vec![IpAddr::from_str("::1").unwrap()]); - } - - #[test] - fn allow_ips_rejects_cidr_instead_of_silently_narrowing() { - // A CIDR range used to be dropped with only a warning, leaving an - // allowlist that excluded every host the operator meant to admit. - let err = parse_allow_ips("10.0.0.0/8").unwrap_err(); - assert!(err.contains("10.0.0.0/8"), "must name the bad entry: {err}"); - assert!(err.contains("CIDR"), "must explain why: {err}"); - } - - #[test] - fn allow_ips_rejects_a_partially_valid_list() { - // One good entry must not mask a typo in another — the result would be - // a narrower allowlist than configured. - assert!(parse_allow_ips("10.0.1.5,not-an-ip").is_err()); - assert!( - parse_allow_ips("localhost").is_err(), - "hostnames unsupported" - ); - } - - #[test] - fn allow_ips_rejects_an_empty_result_that_would_block_everything() { - // An all-invalid list previously produced an empty allowlist, and an - // empty allowlist rejects every connection while the process still - // starts and passes health checks. - let err = parse_allow_ips(" ").unwrap_err(); - assert!(err.contains("reject every connection"), "got {err}"); - assert!(parse_allow_ips(",,,").is_err()); - } - - #[test] - fn allow_ips_tolerates_incidental_whitespace_and_trailing_commas() { - let ips = parse_allow_ips(" 127.0.0.1 , 10.0.0.1 ,").unwrap(); - assert_eq!(ips.len(), 2); - } - - /// `record_command` looks the label up in an immutable, pre-built map. A - /// label `command_name` can produce but the catalog does not list still - /// works — it falls back to an uncached registry lookup — but it pays that - /// lookup on every single call, so the gap should fail CI rather than - /// quietly become a hot-path cost. - #[test] - fn command_name_labels_are_all_pre_registered() { - let labels = [ - "get", - "set", - "del", - "incr", - "decr", - "exists", - "expire", - "ttl", - "type", - "append", - "strlen", - "hset", - "hget", - "hgetall", - "hdel", - "hlen", - "lpush", - "rpush", - "lpop", - "rpop", - "lrange", - "llen", - "sadd", - "srem", - "smembers", - "scard", - "zadd", - "zrange", - "zrem", - "zscore", - "ping", - "auth", - "hello", - "quit", - "client", - "config", - "command", - "scan", - "subscribe", - "publish", - "multi", - "exec", - "watch", - UNKNOWN_COMMAND, - ]; - let missing: Vec<&str> = labels - .into_iter() - .filter(|l| !CMD_COUNTERS.contains_key(l)) - .collect(); - assert!( - missing.is_empty(), - "labels with no pre-built counter (each costs a registry lookup per command): {missing:?}" - ); - } - - /// The counter table is built once from the catalog and never mutated, so - /// concurrent `record_command` calls need no lock and cannot poison one. - /// The previous `RwLock` was `.unwrap()`ed on every command: one panic while - /// holding it poisoned the lock and every later command panicked with it. - #[test] - fn record_command_is_safe_from_many_threads_at_once() { - let threads: Vec<_> = (0..8) - .map(|_| { - std::thread::spawn(|| { - for _ in 0..5_000 { - record_command("get"); - record_command("set"); - record_command(UNKNOWN_COMMAND); - } - }) - }) - .collect(); - for t in threads { - t.join().expect("record_command panicked under contention"); - } - } - - #[test] - fn command_name_is_stable_and_lowercase() { - // These strings become Prometheus label values; renaming one silently - // breaks existing dashboards and alerts. - let cases = [ - (Command::Get("k".into()), "get"), - ( - Command::Set("k".into(), "v".into(), SetOptions::default()), - "set", - ), - (Command::Del(vec!["k".into()]), "del"), - (Command::Incr("k".into()), "incr"), - (Command::HGetAll("k".into()), "hgetall"), - (Command::LPush("k".into(), vec!["v".into()]), "lpush"), - (Command::SAdd("k".into(), vec!["m".into()]), "sadd"), - (Command::Ping(None), "ping"), - ]; - for (cmd, expected) in cases { - assert_eq!(command_name(&cmd), expected, "label drift for {cmd:?}"); - } - } - - // ── Config parsing ──────────────────────────────────────────────────────── - - #[test] - fn parse_memory_bytes_accepts_units_and_bare_numbers() { - assert_eq!(parse_memory_bytes("1024"), Some(1024)); - assert_eq!(parse_memory_bytes("1kb"), Some(1024)); - assert_eq!(parse_memory_bytes("2mb"), Some(2 * 1024 * 1024)); - assert_eq!(parse_memory_bytes("1gb"), Some(1024 * 1024 * 1024)); - } - - #[test] - fn parse_memory_bytes_is_case_and_whitespace_tolerant() { - assert_eq!(parse_memory_bytes(" 2MB "), Some(2 * 1024 * 1024)); - assert_eq!(parse_memory_bytes("2 mb"), Some(2 * 1024 * 1024)); - assert_eq!(parse_memory_bytes("1Gb"), Some(1024 * 1024 * 1024)); - } - - #[test] - fn parse_memory_bytes_rejects_nonsense_rather_than_defaulting() { - // Returning None lets the caller fall back explicitly; silently - // parsing "10 bananas" as 10 bytes would cap memory at nothing. - for bad in ["", "abc", "10 bananas", "-5", "1.5mb", "mb"] { - assert_eq!(parse_memory_bytes(bad), None, "{bad:?} should not parse"); - } - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn integration_ws_sync_scope_filters_fanout() { - let srv = spawn_ws_server().await; - let mut scoped = WsClient::connect(srv.tcp_addr).await; - let mut unscoped = WsClient::connect(srv.tcp_addr).await; - let mut writer = WsClient::connect(srv.tcp_addr).await; - - // Open mode: SYNC with literal patterns. - assert_eq!(scoped.cmd(&["SYNC", "cart:*"]).await, arr(&["cart:*"])); - - assert_eq!(writer.cmd(&["SET", "cart:1", "x"]).await, ok()); - assert_eq!(writer.cmd(&["SET", "other:1", "y"]).await, ok()); - - // Scoped client sees the cart write and nothing else. - let push = scoped.recv_push(1000).await.expect("expected cart:1 push"); - assert!(push.contains("cart:1"), "unexpected push: {push}"); - assert!( - scoped.recv_push(300).await.is_none(), - "out-of-scope push leaked to scoped client" - ); - - // Unscoped client (legacy mode) sees both. - let p1 = unscoped.recv_push(1000).await.expect("push 1"); - let p2 = unscoped.recv_push(1000).await.expect("push 2"); - assert!(p1.contains("cart:1") && p2.contains("other:1")); - - // Bare SYNC reports current scopes. - assert_eq!(scoped.cmd(&["SYNC"]).await, arr(&["cart:*"])); - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn integration_ws_sync_strict_mode_gates_and_filters() { - let secret = "integration-secret"; - let srv = spawn_ws_server_cfg(Some(secret.to_string())).await; - let mut client = WsClient::connect(srv.tcp_addr).await; - - // No token yet: key commands and pushes are refused. - let r = client.cmd(&["GET", "cart:1"]).await; - assert!(matches!(&r, Value::Error(e) if e.contains("NOSCOPE"))); - // Literal patterns are rejected in strict mode. - let r = client.cmd(&["SYNC", "cart:*"]).await; - assert!(matches!(&r, Value::Error(e) if e.contains("signed scopes"))); - // Garbage token. - let r = client.cmd(&["SYNC", "TOKEN", "not-a-token"]).await; - assert!(matches!(&r, Value::Error(e) if e.contains("invalid sync token"))); - - // Valid token: scoped to cart:* only. - let tok = mint_sync_token(secret, "cart:*"); - assert_eq!(client.cmd(&["SYNC", "TOKEN", &tok]).await, arr(&["cart:*"])); - - // In-scope commands work; out-of-scope and admin are refused. - assert_eq!(client.cmd(&["SET", "cart:1", "x"]).await, ok()); - assert_eq!(client.cmd(&["GET", "cart:1"]).await, bulk("x")); - let r = client.cmd(&["GET", "secret-key"]).await; - assert!(matches!(&r, Value::Error(e) if e.contains("NOSCOPE"))); - let r = client.cmd(&["KEYS", "*"]).await; - assert!(matches!(&r, Value::Error(e) if e.contains("NOSCOPE"))); - - // Fan-out: a second scoped client writes in and out of the first's scope. - let mut writer = WsClient::connect(srv.tcp_addr).await; - let wtok = mint_sync_token(secret, "cart:*,other:*"); - assert_eq!( - writer.cmd(&["SYNC", "TOKEN", &wtok]).await, - arr(&["cart:*", "other:*"]) - ); - assert_eq!(writer.cmd(&["SET", "cart:2", "a"]).await, ok()); - assert_eq!(writer.cmd(&["SET", "other:2", "b"]).await, ok()); - - let push = client.recv_push(1000).await.expect("expected cart:2 push"); - assert!(push.contains("cart:2"), "unexpected push: {push}"); - assert!( - client.recv_push(300).await.is_none(), - "out-of-scope push leaked on strict connection" - ); - } - - // ── Exactly-once delivery (DEDUP) ───────────────────────────────────────── - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn integration_dedup_skips_replayed_writes() { - let srv = spawn_ws_server().await; - let mut c = WsClient::connect(srv.tcp_addr).await; - - // First delivery applies. - assert_eq!( - c.cmd(&["DEDUP", "client-a", "1", "INCRBY", "n", "2"]).await, - int(2) - ); - // Exact replay (ack lost, client re-sent) is skipped. - assert_eq!( - c.cmd(&["DEDUP", "client-a", "1", "INCRBY", "n", "2"]).await, - Value::SimpleString("DUP".into()) - ); - // Higher id applies. - assert_eq!( - c.cmd(&["DEDUP", "client-a", "2", "INCRBY", "n", "3"]).await, - int(5) - ); - // The high-water mark survives a reconnect — the whole point. - let mut c2 = WsClient::connect(srv.tcp_addr).await; - assert_eq!( - c2.cmd(&["DEDUP", "client-a", "2", "INCRBY", "n", "3"]) - .await, - Value::SimpleString("DUP".into()) - ); - assert_eq!(srv.store.execute(Command::Get("n".into())), bulk("5")); - // A different client id has an independent mark. - assert_eq!( - c2.cmd(&["DEDUP", "client-b", "1", "INCRBY", "n", "1"]) - .await, - int(6) - ); - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn integration_dedup_respects_sync_scopes() { - let secret = "dedup-secret"; - let srv = spawn_ws_server_cfg(Some(secret.to_string())).await; - let mut c = WsClient::connect(srv.tcp_addr).await; - let tok = mint_sync_token(secret, "cart:*"); - assert_eq!(c.cmd(&["SYNC", "TOKEN", &tok]).await, arr(&["cart:*"])); - - // Scope enforcement applies to the wrapped command. - assert_eq!( - c.cmd(&["DEDUP", "c1", "1", "SET", "cart:1", "x"]).await, - ok() - ); - let r = c.cmd(&["DEDUP", "c1", "2", "SET", "admin:1", "x"]).await; - assert!(matches!(&r, Value::Error(e) if e.contains("NOSCOPE"))); - } - - // ── JSON over the wire ──────────────────────────────────────────────────── - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn integration_json_commands_and_fanout() { - let srv = spawn_ws_server().await; - let mut writer = WsClient::connect(srv.tcp_addr).await; - let mut peer = WsClient::connect(srv.tcp_addr).await; - - assert_eq!( - writer.cmd(&["JSET", "doc:1", "$", r#"{"a":1}"#]).await, - ok() - ); - assert_eq!(writer.cmd(&["JGET", "doc:1", "$.a"]).await, bulk("1")); - assert_eq!( - writer - .cmd(&["JMERGE", "doc:1", r#"{"b":2,"a":null}"#]) - .await, - ok() - ); - assert_eq!(writer.cmd(&["JGET", "doc:1"]).await, bulk(r#"{"b":2}"#)); - - // Peers receive the writes as replayable pushes. - let p = peer.recv_push(1000).await.expect("JSET push"); - assert!(p.contains("JSET") && p.contains("doc:1"), "push: {p}"); - let p2 = peer.recv_push(1000).await.expect("JMERGE push"); - assert!(p2.contains("JMERGE"), "push: {p2}"); - - // Failed writes are not broadcast. - let r = writer.cmd(&["JSET", "doc:1", "$", "{bad"]).await; - assert!(matches!(&r, Value::Error(e) if e.contains("invalid JSON"))); - assert!(peer.recv_push(300).await.is_none()); - } - - // ── Live queries (QSUB / QUNSUB) ────────────────────────────────────────── - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn integration_ws_qsub_initial_state_and_diffs() { - let srv = spawn_ws_server().await; - let mut writer = WsClient::connect(srv.tcp_addr).await; - let mut client = WsClient::connect(srv.tcp_addr).await; - - // Pre-existing state the subscription must deliver up front. - assert_eq!(writer.cmd(&["SET", "cart:1", "apples"]).await, ok()); - assert_eq!(writer.cmd(&["SET", "other:1", "zzz"]).await, ok()); - - let initial = client.cmd(&["QSUB", "cart:*"]).await; - match &initial { - Value::Array(Some(items)) => { - assert_eq!( - items.len(), - 4, - "expected tag + pattern + one pair: {items:?}" - ); - assert_eq!(items[0], bulk("qstate")); - assert_eq!(items[1], bulk("cart:*")); - assert_eq!(items[2], bulk("cart:1")); - assert_eq!(items[3], bulk("apples")); - } - other => panic!("expected initial-state array, got {other:?}"), - } - - // A matching write arrives as a keychange diff… - assert_eq!(writer.cmd(&["SET", "cart:2", "pears"]).await, ok()); - let (key, value) = client.recv_keychange(1000).await.expect("cart:2 diff"); - assert_eq!((key.as_str(), &value), ("cart:2", &bulk("pears"))); - - // …a non-matching write does not… - assert_eq!(writer.cmd(&["SET", "other:2", "yyy"]).await, ok()); - assert!(client.recv_keychange(300).await.is_none()); - - // …a deletion arrives as a nil keychange… - assert_eq!(writer.cmd(&["DEL", "cart:2"]).await, int(1)); - let (key, value) = client.recv_keychange(1000).await.expect("delete diff"); - assert_eq!((key.as_str(), &value), ("cart:2", &nil())); - - // …and QUNSUB stops the stream. - assert_eq!(client.cmd(&["QUNSUB", "cart:*"]).await, ok()); - assert_eq!(writer.cmd(&["SET", "cart:3", "plums"]).await, ok()); - assert!(client.recv_keychange(300).await.is_none()); - } - - #[tokio::test(flavor = "multi_thread", worker_threads = 2)] - async fn integration_ws_qsub_strict_scope() { - let secret = "qsub-secret"; - let srv = spawn_ws_server_cfg(Some(secret.to_string())).await; - let mut client = WsClient::connect(srv.tcp_addr).await; - - let tok = mint_sync_token(secret, "cart:*"); - assert_eq!(client.cmd(&["SYNC", "TOKEN", &tok]).await, arr(&["cart:*"])); - - // A narrower pattern under the grant is allowed (prefix-style cover). - assert_eq!( - client.cmd(&["QSUB", "cart:42:*"]).await, - arr(&["qstate", "cart:42:*"]) - ); - // A pattern outside the grant is refused. - let r = client.cmd(&["QSUB", "admin:*"]).await; - assert!(matches!(&r, Value::Error(e) if e.contains("NOSCOPE"))); - - // Diffs flow for the subscribed pattern. - let mut writer = WsClient::connect(srv.tcp_addr).await; - let wtok = mint_sync_token(secret, "cart:*"); - writer.cmd(&["SYNC", "TOKEN", &wtok]).await; - assert_eq!(writer.cmd(&["SET", "cart:42:item", "x"]).await, ok()); - let (key, _) = client.recv_keychange(1000).await.expect("scoped diff"); - assert_eq!(key, "cart:42:item"); - } -} - -#[cfg(test)] -mod tls_loading_tests { - use super::*; - - // A self-signed cert and its key, generated once with: - // openssl req -x509 -newkey rsa:2048 -keyout k -out c -days 3650 -nodes \ - // -subj "/CN=recached-test" - // Embedded rather than generated at test time so the test needs no openssl - // on the runner and cannot fail for reasons unrelated to parsing. - const TEST_CERT: &str = "-----BEGIN CERTIFICATE-----\nMIIDETCCAfmgAwIBAgIUDpGtGZ5z4j/X0RMdVgiZt5TyukwwDQYJKoZIhvcNAQEL\nBQAwGDEWMBQGA1UEAwwNcmVjYWNoZWQtdGVzdDAeFw0yNjA3MTkxNDUyMDVaFw0z\nNjA3MTYxNDUyMDVaMBgxFjAUBgNVBAMMDXJlY2FjaGVkLXRlc3QwggEiMA0GCSqG\nSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDdi5zyxNocCEi6elQKsS0onYh9aOMW5Hjz\n7zAcWa6EPp1g4Zz1tLF2Nk92CBG/iWzF5OckDChuIYjM+MTRws5UOSXwwkbLplKR\nSMGEst1mP3rZPGHq57w52OmxO599kBR4BpeWhFMC4w5xGEO9Gp4P+QdCIYaUEBxz\nLeEyCwapimzamKRYKO0VoZWzF0bLhYUHxc9FD2QMbaPUmRZZGdcttg/0Gq4U/P5N\n6jhWo+ekIKu1kpLSAZPiHtYNAzGu1sk0lTPyVxdmmwqPueV9MLUgVIpDWA+QL80I\nXIjTfaQAOl4k31AeC+yglCyhB/yl/0ROQUAXGgozsFJnpxujLGMPAgMBAAGjUzBR\nMB0GA1UdDgQWBBSWbJJErt4zE9+u8lbBnAPXaRSI0TAfBgNVHSMEGDAWgBSWbJJE\nrt4zE9+u8lbBnAPXaRSI0TAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUA\nA4IBAQBGslzIW0Q46r7eQGK22fTEfNReSy4f7PZPGn/BZbj499LKSfRP1z8A3bbF\n2CdQKswbhVUbHfLUoaRwRfmJWhR/I/UxNkUfVlQ/jQBaUvg2ZCy1l/3kRM6N1t5K\ntkwg+dzai/6LwT7RHmbl8Dx32on3+x9vJMYtoxeBk4nfHZTQMIOd3zsaXp/+RWUY\nzuIWXX/rf862GerYhoHVCWzMcHMLnI/Mwzlm2tgVnfW1XpI/La3fxnTWYT4g4PIJ\nfXe3WrO9VyC1ZZ7PjE4Pq4unCRbJ2yZ5toybr4kcT4UGFrsXjnAsT+RyLY4By50D\nkaBPsvjq5ZvbiPBtEINXbmF3A7cq\n-----END CERTIFICATE-----"; - const TEST_KEY: &str = "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDdi5zyxNocCEi6\nelQKsS0onYh9aOMW5Hjz7zAcWa6EPp1g4Zz1tLF2Nk92CBG/iWzF5OckDChuIYjM\n+MTRws5UOSXwwkbLplKRSMGEst1mP3rZPGHq57w52OmxO599kBR4BpeWhFMC4w5x\nGEO9Gp4P+QdCIYaUEBxzLeEyCwapimzamKRYKO0VoZWzF0bLhYUHxc9FD2QMbaPU\nmRZZGdcttg/0Gq4U/P5N6jhWo+ekIKu1kpLSAZPiHtYNAzGu1sk0lTPyVxdmmwqP\nueV9MLUgVIpDWA+QL80IXIjTfaQAOl4k31AeC+yglCyhB/yl/0ROQUAXGgozsFJn\npxujLGMPAgMBAAECggEAQ4xj6ClZDx7/fcv6f+ARksalbQdj5gD3V/jfxGUbrrqg\npX9kqg3T5eUdSTGgp7Ow9I2cZANI+HtFCKn46LPq0QczqDqz9zfZCO8UAe+/TYOh\nY0bj3AmX/FNEvYMeV9xsQURRR9VEsiakqprpXGkXNGuLaQBr1g0rf3rHpMhz2ZEH\nw7QxoUfH3YL7fMIWwAvHanl6HwzE3TVh3felFGGGiqUaGg2Pll+/s5AiYnnZuq39\nW+t0RVH36rSFh037Su/ScCs44WZS+kGyqcuxyWLwvNwXEVXC3h42N62MHGgY8xQw\nP8wMSxGejEIr8IGpwhl86+oW+44nxarmrMBzMPRIcQKBgQD6VnpsIwxWV1zNC1LD\nbTVRQOoqJWDzxdXB47IB29ADHJnfBLbMr/6kIgIfuCu1Kf9wWTZGHwFUOKFhJDBC\ngLHpWFMWKSew4UmNyc0a16a9Pb7mdNyxnTY1oQucEbzS3pLMzda2dAdxYJE0Cuc6\nJ8Xp2Jo73LnRxY+NyEyl28frAwKBgQDijmtG0cDESNPMhxqJO/9KoRqRT3T+JqNf\noWaKlfSlQFaGecjdk9dNPZ1Aew4xI0v/C5YTwT6MUVDEXmoaSsa+S1atoDLNsojL\nuWqUno9mF6o3U23pi4vlEYh6c/V7Bd1VYde8ZQqVq0KxbCmYp1VE+DBeyNNT7Q5X\nN2lst0hEBQKBgQCXjds3tFA3xVQNXpmQboEk2+Pn+BEmA9NROoP91BGukJYnCjeQ\n28uRmnUmttzfJLncTmYpNYQcdNxebwY4fKk415wVgnzg/MMG7/EYGw566vKzmnQx\noze6Z/EbXzGth8nf643dj4kh/pBprWAnOQT8eYGGVC667Jvn/idJEjGJ+QKBgQCz\nGmgQio3cHr7huATwbO/7rbT1H12b9iu91DjeYoIPifddRDXZhaD1vTnt2dp0WjUg\nIaa5Y1HxV++D7ifvNSI9Gg4iIL1JBFVEyQZLC7bNvPOh3WDM+rbTlrLQK4/re81o\nTHtiwnZFsCh/XsTbm527coG6zQTUGln19SZw/cwxiQKBgA8dcEyBvPi6JgAqLy+5\n3Ev1uZEKkAeAQAkOV9jzqDN9NTi7GWOz3mtY2zopYjef7Wl0V4Qjkr7Jkxlx2wyn\nHboOuCEjComkRxn5vrHm6EBp0uTrdFIknLysxmQFgNamp9E8mX9p/q9rq7aZWzPu\nr+3jOYvwFyzAQ4j2tGzUm7Zd\n-----END PRIVATE KEY-----"; - - fn write(name: &str, body: &str) -> std::path::PathBuf { - let path = - std::env::temp_dir().join(format!("recached_test_{name}_{}", std::process::id())); - std::fs::write(&path, body).expect("write pem"); - path - } - - // ── replication TLS (2.2) ─────────────────────────────────────────────── - - #[test] - fn the_verified_servername_defaults_to_the_primarys_host() { - // What an operator means by "connect to this primary" is the host they - // named, so that is what the certificate is checked against. - assert_eq!( - repl_tls_servername("primary.internal:6381", None), - "primary.internal" - ); - assert_eq!(repl_tls_servername("10.0.1.5:6381", None), "10.0.1.5"); - assert_eq!( - repl_tls_servername("primary.internal", None), - "primary.internal" - ); - } - - #[test] - fn an_ipv6_primary_address_is_not_split_inside_the_address() { - // Splitting on the last colon would cut inside an IPv6 literal and - // produce a servername that could never validate. - assert_eq!(repl_tls_servername("[::1]:6381", None), "::1"); - assert_eq!(repl_tls_servername("[fd00::5]:6381", None), "fd00::5"); - } - - #[test] - fn the_servername_override_wins_and_ignores_blanks() { - // The common deployment points RECACHED_REPLICAOF at an IP while the - // certificate names a host, which cannot validate without an IP SAN. - assert_eq!( - repl_tls_servername("10.0.1.5:6381", Some("primary.internal".into())), - "primary.internal" - ); - assert_eq!( - repl_tls_servername("10.0.1.5:6381", Some(" primary.internal ".into())), - "primary.internal" - ); - // Empty or whitespace is "unset", not "verify against nothing". - assert_eq!( - repl_tls_servername("10.0.1.5:6381", Some(String::new())), - "10.0.1.5" - ); - assert_eq!( - repl_tls_servername("10.0.1.5:6381", Some(" ".into())), - "10.0.1.5" - ); - } - - #[test] - fn a_missing_or_unusable_replication_ca_is_a_startup_error() { - // Trusting nothing would fail every connection at runtime rather than at - // startup, which is much harder to diagnose from a replica's logs. - // `TlsConnector` is not Debug, so unwrap the error side by hand. - let missing = load_repl_tls_connector("/nonexistent/recached-ca.pem"); - let Err(err) = missing else { - panic!("a missing CA file must be refused"); - }; - assert!(err.contains("RECACHED_REPL_TLS_CA"), "{err}"); - - let junk = write("repl_ca_junk.pem", "not a certificate\n"); - let unusable = load_repl_tls_connector(junk.to_str().unwrap()); - let _ = std::fs::remove_file(&junk); - let Err(err) = unusable else { - panic!("a file with no certificates must be refused"); - }; - assert!(err.contains("RECACHED_REPL_TLS_CA"), "{err}"); - } - - #[test] - fn a_self_signed_certificate_works_as_the_replication_trust_anchor() { - // Pinning the primary's own certificate is the documented path, and the - // reason the trust anchor is an explicit file rather than the system root - // store: replication is a private link between two hosts one operator - // runs, so trusting every public CA to vouch for it would be backwards. - let ca = write("repl_ca_ok.pem", TEST_CERT); - assert!( - load_repl_tls_connector(ca.to_str().unwrap()).is_ok(), - "a valid self-signed PEM must build a connector" - ); - let _ = std::fs::remove_file(&ca); - } - - #[test] - fn a_pem_certificate_and_key_load() { - // PEM parsing moved from the deprecated rustls-pemfile to - // rustls-pki-types. These two functions had no coverage at all, so the - // swap would have been verified only by the code compiling. - let cert_path = write("tls_load.crt", TEST_CERT); - let key_path = write("tls_load.key", TEST_KEY); - - let certs = load_certs(cert_path.to_str().unwrap()).expect("cert must parse"); - assert_eq!(certs.len(), 1, "one certificate in the chain"); - assert!(!certs[0].as_ref().is_empty(), "DER body must be non-empty"); - - let key = load_private_key(key_path.to_str().unwrap()).expect("key must parse"); - assert!(!key.secret_der().is_empty(), "key DER must be non-empty"); - - let _ = std::fs::remove_file(&cert_path); - let _ = std::fs::remove_file(&key_path); - } - - #[test] - fn a_missing_file_is_an_error_not_a_panic() { - assert!(load_certs("/nonexistent/recached-test.crt").is_err()); - assert!(load_private_key("/nonexistent/recached-test.key").is_err()); - } - - #[test] - fn a_file_with_no_pem_content_is_rejected() { - // Pointing RECACHED_TLS_CERT at the wrong file must fail loudly rather - // than yielding an empty chain that rustls would later reject with a - // much less obvious error. - let junk = write("tls_junk.crt", "this is not a PEM file\n"); - assert!( - load_certs(junk.to_str().unwrap()) - .map(|c| c.is_empty()) - .unwrap_or(true), - "non-PEM input must not yield certificates" - ); - let junk_key = write("tls_junk.key", "still not PEM\n"); - assert!(load_private_key(junk_key.to_str().unwrap()).is_err()); - - let _ = std::fs::remove_file(&junk); - let _ = std::fs::remove_file(&junk_key); - } -} - -#[cfg(test)] -mod limit_config_tests { - use super::*; - - /// Serialises the tests in this module. - /// - /// Environment variables are process-global and `cargo test` runs tests on - /// parallel threads, so a test that sets `RECACHED_MAX_*` races any test - /// reading the same variable — which is why `set_var` is `unsafe`. This - /// surfaced as `compiled_defaults_match_the_documented_values` - /// intermittently observing an override (`7`) instead of a default - /// (`10_000`): it passed locally and failed in CI purely on thread timing. - /// - /// Poisoning is ignored deliberately: one failing test must not cascade - /// into unrelated failures in the rest of the module. - static ENV_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); - - fn env_guard() -> std::sync::MutexGuard<'static, ()> { - ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()) - } - - #[test] - fn env_limit_falls_back_to_the_default() { - let _guard = env_guard(); - // Unset, empty, non-numeric, and zero all mean "use the default" — - // a zero limit would disable the feature rather than tune it. - assert_eq!(env_limit("RECACHED_DEFINITELY_UNSET_VAR_XYZ", 64), 64); - for bad in ["", " ", "abc", "0", "-5", "1.5"] { - unsafe { std::env::set_var("RECACHED_TEST_LIMIT", bad) }; - assert_eq!(env_limit("RECACHED_TEST_LIMIT", 64), 64, "input {bad:?}"); - } - unsafe { std::env::remove_var("RECACHED_TEST_LIMIT") }; - } - - #[test] - fn env_limit_accepts_a_positive_override() { - let _guard = env_guard(); - unsafe { std::env::set_var("RECACHED_TEST_LIMIT_OK", " 256 ") }; - assert_eq!( - env_limit("RECACHED_TEST_LIMIT_OK", 64), - 256, - "whitespace tolerated" - ); - unsafe { std::env::remove_var("RECACHED_TEST_LIMIT_OK") }; - } - - #[test] - fn overrides_are_read_from_the_documented_variable_names() { - let _guard = env_guard(); - // A bulk rename once rewrote these string literals along with the - // function names, leaving variables like `RECACHED_max_watches_per_conn()` - // that no operator would ever set — the override silently did nothing. - // Assert the names the docs promise. - for (var, default) in [ - ("RECACHED_MAX_MULTI_QUEUE", 10_000usize), - ("RECACHED_MAX_WATCHES_PER_CONN", 1_024), - ("RECACHED_MAX_LIVE_QUERIES", 64), - ("RECACHED_MAX_QSUB_INITIAL_KEYS", 10_000), - ("RECACHED_EVICTION_SAMPLE", 10), - ] { - assert!( - var.chars() - .all(|c| c.is_ascii_uppercase() || c == '_' || c.is_ascii_digit()), - "{var} is not a plausible environment variable name" - ); - unsafe { std::env::set_var(var, "7") }; - assert_eq!(env_limit(var, default), 7, "{var} override ignored"); - unsafe { std::env::remove_var(var) }; - } - } - - #[test] - fn compiled_defaults_match_the_documented_values() { - let _guard = env_guard(); - // These appear in docs/server/operations.md; drift would mislead - // operators sizing a deployment. - assert_eq!(max_multi_queue_len(), 10_000); - assert_eq!(max_watches_per_conn(), 1_024); - assert_eq!(max_qsubs_per_conn(), 64); - assert_eq!(max_qsub_initial_keys(), 10_000); - } -} - -// ───────────────────────────────────────────────────────────────────────────── -// Hardening: the network-exposure fixes and their decision functions. -// -// Every check here guards a boundary that was open in 0.2.4 or earlier. The -// pure-function shape is deliberate: the replication gate and the origin -// allowlist are decisions, and a decision can be asserted without standing up a -// listener or mutating process-global environment state. -// ───────────────────────────────────────────────────────────────────────────── -#[cfg(test)] -mod hardening_tests { - use super::*; - - // ── replication listener gate ───────────────────────────────────────────── - - #[test] - fn the_replication_port_stays_closed_unless_asked_for() { - // The default. Before this gate existed the listener bound - // 0.0.0.0:6381 on every node and, with no password, served the entire - // keyspace to anyone who connected — so an operator who set - // RECACHED_PASSWORD was protecting nothing. - assert_eq!(resolve_repl_listen(None, "0.0.0.0", None), Ok(false)); - assert_eq!(resolve_repl_listen(None, "0.0.0.0", Some("pw")), Ok(false)); - // An empty value is "unset", not "true". - assert_eq!( - resolve_repl_listen(Some(String::new()), "0.0.0.0", None), - Ok(false) - ); - assert_eq!( - resolve_repl_listen(Some(" ".to_string()), "0.0.0.0", None), - Ok(false) - ); - } - - #[test] - fn enabling_it_on_a_public_interface_without_a_password_refuses_to_start() { - let err = resolve_repl_listen(Some("1".into()), "0.0.0.0", None) - .expect_err("public + no password must not be allowed"); - // The message has to name both variables — an operator reading a log - // line needs to know what to set, not merely that something is wrong. - assert!(err.contains("RECACHED_REPL_PASSWORD"), "{err}"); - assert!(err.contains("RECACHED_REPL_ENABLE"), "{err}"); - assert!(err.contains("0.0.0.0"), "{err}"); - - // A specific LAN address is just as reachable as 0.0.0.0. - assert!(resolve_repl_listen(Some("1".into()), "10.0.1.5", None).is_err()); - // So is a hostname we cannot resolve to a loopback address: the - // conservative reading is the one that demands a password. - assert!(resolve_repl_listen(Some("1".into()), "cache.internal", None).is_err()); - // An empty password is not a password. - assert!(resolve_repl_listen(Some("1".into()), "0.0.0.0", Some("")).is_err()); - } - - #[test] - fn enabling_it_is_allowed_on_loopback_or_with_a_password() { - // Loopback without a password is a development setup, not an exposure. - assert_eq!( - resolve_repl_listen(Some("1".into()), "127.0.0.1", None), - Ok(true) - ); - assert_eq!( - resolve_repl_listen(Some("yes".into()), "::1", None), - Ok(true) - ); - assert_eq!( - resolve_repl_listen(Some("on".into()), "localhost", None), - Ok(true) - ); - // Public is fine once authenticated — this is the multi-tier - // replication path, which must keep working. - assert_eq!( - resolve_repl_listen(Some("true".into()), "0.0.0.0", Some("pw")), - Ok(true) - ); - assert_eq!( - resolve_repl_listen(Some("1".into()), "10.0.1.5", Some("pw")), - Ok(true) - ); - } - - #[test] - fn an_ambiguous_enable_value_refuses_to_start() { - // Treating `please` as false would leave an operator believing - // replication was on; treating it as true would open a port nobody - // asked for. Neither is acceptable for a variable gating a boundary. - let err = resolve_repl_listen(Some("please".into()), "127.0.0.1", None).unwrap_err(); - assert!(err.contains("RECACHED_REPL_ENABLE"), "{err}"); - assert!(err.contains("not a boolean"), "{err}"); - } - - #[test] - fn boolean_env_values_cover_the_conventional_spellings() { - for yes in ["1", "true", "TRUE", "yes", "On", " on "] { - assert_eq!(parse_env_bool("V", yes), Ok(true), "{yes:?}"); - } - for no in ["0", "false", "FALSE", "no", "Off", " off "] { - assert_eq!(parse_env_bool("V", no), Ok(false), "{no:?}"); - } - assert!(parse_env_bool("V", "maybe").is_err()); - } - - #[test] - fn loopback_detection_treats_unparseable_hosts_as_public() { - assert!(bind_is_loopback("127.0.0.1")); - assert!(bind_is_loopback("127.0.0.53")); - assert!(bind_is_loopback("::1")); - assert!(bind_is_loopback("localhost")); - assert!(bind_is_loopback("LOCALHOST")); - assert!(!bind_is_loopback("0.0.0.0")); - assert!(!bind_is_loopback("10.0.1.5")); - assert!(!bind_is_loopback("::")); - assert!(!bind_is_loopback("cache.internal")); - assert!(!bind_is_loopback("")); - // An IPv6 bind address must be written bracketed for the listeners to - // format it correctly, so the brackets have to be tolerated here too — - // otherwise `[::1]` is misread as public and demands a password. - assert!(bind_is_loopback("[::1]")); - assert!(!bind_is_loopback("[::]")); - assert!(!bind_is_loopback("[fd00::5]")); - } - - // ── replication auth: throttle and handshake ───────────────────────────── - - #[test] - fn repeated_bad_replication_passwords_block_the_peer() { - // The RESP port drops a connection after five guesses, but the - // replication handshake is one-shot: reconnecting used to reset the - // count, so the port offered unlimited guesses at a secret that yields - // the whole keyspace. The throttle is keyed by address for that reason. - let throttle = ReplAuthThrottle::new(); - let ip = IpAddr::from([203, 0, 113, 7]); - assert!(!throttle.is_blocked(ip)); - for _ in 0..MAX_AUTH_FAILURES { - assert!(!throttle.is_blocked(ip), "must not block before the cap"); - throttle.record_failure(ip); - } - assert!(throttle.is_blocked(ip), "cap reached, peer must be refused"); - - // Other peers are unaffected — one attacker must not lock out a fleet. - assert!(!throttle.is_blocked(IpAddr::from([203, 0, 113, 8]))); - - // A successful handshake clears the record. - throttle.record_success(ip); - assert!(!throttle.is_blocked(ip)); - } - - #[test] - fn the_throttle_does_not_grow_without_bound() { - // A spray from many source addresses must not be a memory-growth - // vector; the map sweeps once it crosses the threshold. - let throttle = ReplAuthThrottle::new(); - for i in 0..(REPL_AUTH_SWEEP_THRESHOLD + 64) { - let ip = IpAddr::from([ - 10, - ((i >> 16) & 0xff) as u8, - ((i >> 8) & 0xff) as u8, - (i & 0xff) as u8, - ]); - throttle.record_failure(ip); - } - let len = throttle.failures.lock().unwrap().len(); - // Entries are all fresh so none are swept, but the sweep must have run - // without panicking and the map must stay proportional to the input - // rather than duplicating it. - assert!(len <= REPL_AUTH_SWEEP_THRESHOLD + 64, "{len}"); - } - - #[tokio::test] - async fn the_auth_line_is_read_to_its_terminator_not_to_the_password_length() { - // Reading exactly `password.len() + 1` bytes made the number of bytes - // the server waited for *be* the password length, recoverable by - // drip-feeding one byte at a time. - let (mut client, mut server) = tokio::io::duplex(256); - client.write_all(b"hunter2\n").await.unwrap(); - let line = read_repl_auth_line(&mut server).await.unwrap(); - assert_eq!(line, b"hunter2"); - } - - #[tokio::test] - async fn an_auth_line_without_a_terminator_is_refused_at_the_cap() { - let (mut client, mut server) = tokio::io::duplex(4096); - let flood = vec![b'x'; MAX_REPL_AUTH_LINE + 16]; - // Write concurrently: the reader gives up mid-stream, so the writer - // must not block on a full pipe. - tokio::spawn(async move { - let _ = client.write_all(&flood).await; - }); - let err = read_repl_auth_line(&mut server) - .await - .expect_err("an unterminated line must not be read forever"); - assert_eq!(err.kind(), ErrorKind::InvalidData); - } - - #[tokio::test] - async fn a_short_auth_line_still_compares_unequal() { - // The comparison is constant-time and length-checked, so a truncated - // guess fails rather than matching a prefix. - let (mut client, mut server) = tokio::io::duplex(256); - client.write_all(b"hunt\n").await.unwrap(); - let line = read_repl_auth_line(&mut server).await.unwrap(); - assert!(!ct_eq_bytes(&line, b"hunter2")); - } - - // ── WebSocket origin allowlist ────────────────────────────────────────── - - #[test] - fn an_unset_origin_allowlist_permits_everything() { - // Matches how an unset RECACHED_PASSWORD behaves. The project ships - // insecure-by-default deliberately and says so; what it must not do is - // ship a *silent* default, hence the startup warning. - assert!(origin_allowed(None, Some("https://evil.example"))); - assert!(origin_allowed(None, None)); - } - - #[test] - fn a_foreign_origin_is_refused_when_the_allowlist_is_set() { - // The finding this closes: browsers apply neither CORS nor a preflight - // to WebSockets, so without this check any page a user visits could - // open a socket to ws://localhost:6380 and read or write every key. - let allow = vec!["https://app.example.com".to_string()]; - assert!(origin_allowed( - Some(&allow), - Some("https://app.example.com") - )); - assert!(!origin_allowed(Some(&allow), Some("https://evil.example"))); - // A different scheme or port is a different origin. - assert!(!origin_allowed( - Some(&allow), - Some("http://app.example.com") - )); - assert!(!origin_allowed( - Some(&allow), - Some("https://app.example.com:8443") - )); - // Substring matching would be a hole: `app.example.com.evil.test` - // contains an allowlisted origin as a prefix. - assert!(!origin_allowed( - Some(&allow), - Some("https://app.example.com.evil.test") - )); - } - - #[test] - fn an_absent_origin_is_permitted_because_only_browsers_send_one() { - // A native client omits the header and an attacker with a socket can - // forge it, so refusing here would break legitimate clients while - // stopping nobody. The control exists to separate "the app I deployed" - // from "another page in the same browser". - let allow = vec!["https://app.example.com".to_string()]; - assert!(origin_allowed(Some(&allow), None)); - } - - #[test] - fn origin_comparison_ignores_case_and_a_trailing_slash() { - let allow = parse_allowed_origins("https://App.Example.com/").unwrap(); - assert!(origin_allowed( - Some(&allow), - Some("https://app.example.com") - )); - assert!(origin_allowed( - Some(&allow), - Some("HTTPS://APP.EXAMPLE.COM/") - )); - } - - #[test] - fn the_origin_allowlist_parses_a_list_and_admits_null() { - let list = parse_allowed_origins( - "https://app.example.com, http://localhost:3000 ,https://admin.example.com:8443", - ) - .unwrap(); - assert_eq!( - list, - vec![ - "https://app.example.com", - "http://localhost:3000", - "https://admin.example.com:8443", - ] - ); - // Sandboxed iframes and file:// documents send the literal `null`. - assert_eq!(parse_allowed_origins("null").unwrap(), vec!["null"]); - assert!(origin_allowed( - Some(&parse_allowed_origins("null").unwrap()), - Some("null") - )); - } - - #[test] - fn the_origin_allowlist_rejects_entries_that_could_never_match() { - // Each of these would parse into something a browser never sends, so - // the allowlist would silently reject every connection. Failing at - // startup is the only way an operator finds out. - for bad in [ - "app.example.com", - "https://app.example.com/dashboard", - "://nohost", - "https://", - ] { - assert!( - parse_allowed_origins(bad).is_err(), - "{bad:?} should be rejected" - ); - } - // Set-but-empty would reject every browser; unset is how you allow all. - let err = parse_allowed_origins(" , ").unwrap_err(); - assert!(err.contains("Unset it"), "{err}"); - } - - // ── handshake deadline ────────────────────────────────────────────────── - - #[tokio::test] - async fn a_stalled_websocket_handshake_gives_up_and_releases_the_socket() { - // The connection permit is acquired *before* the handshake runs, so - // without this deadline `RECACHED_MAX_CONNECTIONS` sockets that connect - // and then say nothing — costing an attacker nothing — hold every slot - // indefinitely and the server stops accepting real clients. - let (_client, server) = tokio::io::duplex(1024); - let start = std::time::Instant::now(); - let out = ws_handshake(server, None, Duration::from_millis(150), 1).await; - assert!(out.is_none(), "a silent peer must not produce a stream"); - assert!( - start.elapsed() < Duration::from_secs(2), - "gave up after {:?} — the deadline did not apply", - start.elapsed() - ); - } - - #[test] - fn the_handshake_deadline_has_a_documented_default() { - assert_eq!(DEFAULT_HANDSHAKE_TIMEOUT_SECS, 10); - } - - // ── persistence file permissions ──────────────────────────────────────── - - #[tokio::test] - #[cfg(unix)] - async fn snapshot_and_sidecar_files_are_not_world_readable() { - use std::os::unix::fs::PermissionsExt; - - let dir = std::env::temp_dir().join(format!("recached_perm_{}", std::process::id())); - tokio::fs::create_dir_all(&dir).await.unwrap(); - let path = dir.join("perm-test.rdb"); - - write_private(&path, b"payload").await.unwrap(); - let mode = tokio::fs::metadata(&path) - .await - .unwrap() - .permissions() - .mode(); - assert_eq!( - mode & 0o777, - 0o600, - "snapshots are plaintext dumps of the keyspace; 0644 lets any local user read the cache" - ); - - // A file left behind 0644 by an earlier version must be tightened on - // the next write, not keep its old mode forever. - tokio::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o644)) - .await - .unwrap(); - write_private(&path, b"payload2").await.unwrap(); - let mode = tokio::fs::metadata(&path) - .await - .unwrap() - .permissions() - .mode(); - assert_eq!(mode & 0o777, 0o600, "an existing loose mode must be fixed"); - assert_eq!(tokio::fs::read(&path).await.unwrap(), b"payload2"); - - let _ = tokio::fs::remove_dir_all(&dir).await; - } - - #[tokio::test] - #[cfg(unix)] - async fn the_aof_is_not_world_readable() { - use std::os::unix::fs::PermissionsExt; - - let dir = std::env::temp_dir().join(format!("recached_aofperm_{}", std::process::id())); - tokio::fs::create_dir_all(&dir).await.unwrap(); - let path = dir.join("perm-test.aof"); - - // Pre-create it loose, as an upgrade from an earlier version would. - tokio::fs::write(&path, b"").await.unwrap(); - tokio::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o644)) - .await - .unwrap(); - - let writer = AofWriter::open(path.clone(), AofSync::No).await.unwrap(); - writer.append(b"*1\r\n$4\r\nPING\r\n").await; - let mode = tokio::fs::metadata(&path) - .await - .unwrap() - .permissions() - .mode(); - assert_eq!(mode & 0o777, 0o600); - - let _ = tokio::fs::remove_dir_all(&dir).await; - } - - #[test] - fn temp_files_do_not_collide_between_processes() { - // A fixed `.tmp` name meant two servers sharing a data directory would - // clobber each other's half-written snapshot. - let a = temp_sibling(std::path::Path::new("/data/recached.rdb"), "snap"); - assert!( - a.to_string_lossy() - .contains(&std::process::id().to_string()), - "{a:?}" - ); - assert!(a.to_string_lossy().ends_with(".tmp"), "{a:?}"); - assert_ne!( - a, - temp_sibling(std::path::Path::new("/data/recached.rdb"), "dedup") - ); - } -} - -// ───────────────────────────────────────────────────────────────────────────── -// Durability: the guarantees `RECACHED_AOF_SYNC` and snapshot saving advertise. -// -// These assert reachability and effect rather than device-level durability — a -// unit test cannot pull the power. What they pin is that the fsync path is -// actually taken, that it does not corrupt or lose data, and that the pieces a -// crash-consistency argument depends on (temp file synced before rename, parent -// directory synced after) are wired up. -// ───────────────────────────────────────────────────────────────────────────── -#[cfg(test)] -mod durability_tests { - use super::*; - - fn scratch(name: &str) -> PathBuf { - let dir = std::env::temp_dir().join(format!( - "recached_dur_{}_{}_{}", - name, - std::process::id(), - next_conn_id() - )); - std::fs::create_dir_all(&dir).expect("create scratch dir"); - dir - } - - #[tokio::test] - async fn aof_always_survives_a_reopen_with_every_byte_intact() { - // `always` previously called flush(), which reaches the page cache and - // not the device. The observable part of the fix is that the fsync path - // runs and is still byte-exact. - let dir = scratch("aof_always"); - let path = dir.join("a.aof"); - let w = AofWriter::open(path.clone(), AofSync::Always) - .await - .unwrap(); - for i in 0..64 { - w.append(format!("*1\r\n${}\r\n{}\r\n", i.to_string().len(), i).as_bytes()) - .await; - } - drop(w); - - let on_disk = tokio::fs::read(&path).await.unwrap(); - let expected: Vec = (0..64) - .flat_map(|i| format!("*1\r\n${}\r\n{}\r\n", i.to_string().len(), i).into_bytes()) - .collect(); - assert_eq!(on_disk, expected, "fsync must not disturb the byte stream"); - - let _ = tokio::fs::remove_dir_all(&dir).await; - } - - #[tokio::test] - async fn aof_everysec_flush_is_idempotent_and_lossless() { - // The everysec ticker calls flush() on a cadence, including when nothing - // has been appended since the last tick. - let dir = scratch("aof_everysec"); - let path = dir.join("b.aof"); - let w = AofWriter::open(path.clone(), AofSync::EverySec) - .await - .unwrap(); - w.append(b"*1\r\n$4\r\nPING\r\n").await; - w.flush().await; - w.flush().await; // nothing new to sync - assert_eq!( - tokio::fs::read(&path).await.unwrap(), - b"*1\r\n$4\r\nPING\r\n" - ); - let _ = tokio::fs::remove_dir_all(&dir).await; - } - - #[tokio::test] - async fn truncating_the_aof_leaves_it_empty_and_reusable() { - // Truncation follows a snapshot, and is now fsynced so a crash cannot - // resurrect a log the snapshot already subsumed. It must also leave the - // handle writable — the server keeps appending to it afterwards. - let dir = scratch("aof_trunc"); - let path = dir.join("c.aof"); - let w = AofWriter::open(path.clone(), AofSync::No).await.unwrap(); - w.append(b"*1\r\n$4\r\nPING\r\n").await; - w.truncate().await; - assert_eq!(tokio::fs::metadata(&path).await.unwrap().len(), 0); - - w.append(b"*1\r\n$4\r\nECHO\r\n").await; - w.flush().await; - assert_eq!( - tokio::fs::read(&path).await.unwrap(), - b"*1\r\n$4\r\nECHO\r\n", - "the writer must still be usable after truncation" - ); - let _ = tokio::fs::remove_dir_all(&dir).await; - } - - #[tokio::test] - async fn a_snapshot_lands_atomically_and_leaves_no_temp_file() { - // The temp file is fsynced, renamed over the target, and then the - // directory is fsynced. A leftover temp file would mean the rename never - // happened, which is the failure this sequence exists to prevent. - let dir = scratch("snap"); - let path = dir.join("dump.rdb"); - let store = KeyValueStore::new(); - for i in 0..32 { - store.execute(Command::Set( - format!("k{i}"), - format!("v{i}").into_bytes(), - Default::default(), - )); - } - let cfg = SnapshotConfig { - path: path.clone(), - last_save: AtomicI64::new(0), - }; - save_snapshot(&store, &cfg).await; - - assert!(path.exists(), "snapshot must exist after save"); - assert!( - cfg.last_save.load(Ordering::Relaxed) > 0, - "a completed save must advance LASTSAVE" - ); - - let leftovers: Vec<_> = std::fs::read_dir(&dir) - .unwrap() - .filter_map(Result::ok) - .map(|e| e.file_name().to_string_lossy().into_owned()) - .filter(|n| n.ends_with(".tmp")) - .collect(); - assert!( - leftovers.is_empty(), - "temp files left behind: {leftovers:?}" - ); - - // And the bytes are a snapshot we can actually read back. - let restored = KeyValueStore::new(); - assert!(load_snapshot(&restored, &path).await); - assert_eq!( - restored.execute(Command::Get("k7".into())), - Value::BulkString(Some(b"v7".to_vec())) - ); - - let _ = tokio::fs::remove_dir_all(&dir).await; - } - - #[tokio::test] - async fn syncing_a_parent_directory_tolerates_odd_paths() { - // Best-effort by contract: a path with no parent, or one that does not - // exist, must warn rather than panic or hang. The snapshot path is - // frequently relative ("recached.rdb"), which has an empty parent. - sync_parent_dir(std::path::Path::new("recached.rdb")).await; - sync_parent_dir(std::path::Path::new("/")).await; - sync_parent_dir(std::path::Path::new("/nonexistent-recached-dir/x.rdb")).await; - } - - #[test] - fn a_sync_token_cannot_grant_an_over_long_pattern() { - // Token patterns reach glob_match without passing through the command - // parser, so the cap has to be repeated there. These are matched once - // per key per write — the most expensive place a pattern can sit. - use base64::Engine as _; - use hmac::{Hmac, Mac}; - use sha2::Sha256; - - let secret = "s3cret"; - let engine = base64::engine::general_purpose::URL_SAFE_NO_PAD; - let mint = |payload: &str| { - let p = engine.encode(payload); - let mut mac = Hmac::::new_from_slice(secret.as_bytes()).unwrap(); - mac.update(p.as_bytes()); - format!("{}.{}", p, engine.encode(mac.finalize().into_bytes())) - }; - - let long = "a".repeat(core_engine::store::MAX_PATTERN_BYTES + 1); - let err = verify_sync_token(secret, &mint(&long)) - .expect_err("an over-long granted pattern must be refused"); - assert_eq!(err, "token grants an over-long pattern"); - - // One over-long pattern in an otherwise fine list is still refused. - assert!(verify_sync_token(secret, &mint(&format!("cart:*,{long}"))).is_err()); - - // A pattern exactly at the cap is still honoured. - let at_cap = "a".repeat(core_engine::store::MAX_PATTERN_BYTES); - assert_eq!( - verify_sync_token(secret, &mint(&at_cap)), - Ok(vec![at_cap.clone()]) - ); - assert_eq!( - verify_sync_token(secret, &mint("cart:42:*,user:1:*")), - Ok(vec!["cart:42:*".to_string(), "user:1:*".to_string()]) - ); - } -} +// ── TCP handler ─────────────────────────────────────────────────────────────── diff --git a/server-native/src/persistence.rs b/server-native/src/persistence.rs new file mode 100644 index 0000000..93e8814 --- /dev/null +++ b/server-native/src/persistence.rs @@ -0,0 +1,457 @@ +//! Durability: RDB-style snapshots and the append-only file, plus the +//! private-permission file helpers both rely on. + +use crate::*; + +/// Write `bytes` to `path`, creating it readable only by this user. +/// +/// Snapshots, the AOF, and the dedup sidecar are plaintext MessagePack dumps of +/// the keyspace. `fs::write` creates with the process umask — `0644` on a +/// typical host — so any local user could read the entire cache. The +/// documentation told operators to protect these files with filesystem +/// permissions; the server should never have been relying on that. +/// +/// Permissions are also set explicitly after opening, so a file left behind +/// `0644` by an earlier version is tightened on the next write rather than +/// keeping its old mode forever. +/// Writes are fsynced before returning. Every caller is writing state that has +/// to survive a crash — a snapshot about to be renamed into place, or the dedup +/// high-water marks that stop a replayed write being applied twice — so the +/// barrier belongs here rather than at each call site. +pub(crate) async fn write_private(path: &std::path::Path, bytes: &[u8]) -> std::io::Result<()> { + let mut opts = tokio::fs::OpenOptions::new(); + opts.write(true).create(true).truncate(true); + #[cfg(unix)] + opts.mode(0o600); + let mut f = opts.open(path).await?; + #[cfg(unix)] + restrict_permissions(&f).await; + f.write_all(bytes).await?; + f.flush().await?; + // `sync_all`, not `sync_data`: this file was just created, so its metadata + // is part of what has to reach the device. + f.sync_all().await?; + Ok(()) +} + +/// fsync the directory holding `path`, making a `rename` into it durable. +/// +/// Renaming a fsynced temp file over the target is atomic with respect to +/// readers, but the *directory entry* is itself just a write: without this, a +/// crash can leave the old file, or no file, despite the new contents being +/// safely on disk. Only meaningful on unix — Windows has no directory handle to +/// sync — so the call is compiled out elsewhere. +#[cfg(unix)] +pub(crate) async fn sync_parent_dir(path: &std::path::Path) { + let Some(dir) = path.parent() else { + return; + }; + // An empty parent means the path was relative with no directory component. + let dir = if dir.as_os_str().is_empty() { + std::path::Path::new(".") + } else { + dir + }; + match tokio::fs::File::open(dir).await { + Ok(f) => { + if let Err(e) = f.sync_all().await { + warn!("Directory fsync failed for {:?}: {}", dir, e); + } + } + Err(e) => warn!("Could not open {:?} to fsync: {}", dir, e), + } +} + +#[cfg(not(unix))] +pub(crate) async fn sync_parent_dir(_path: &std::path::Path) {} + +/// Tighten an already-open file to `0600`, ignoring failure. +/// +/// Best-effort by design: on a filesystem that cannot represent unix modes this +/// is not something to fail a write over, and the caller has already created the +/// file with the right mode where the platform allows it. +#[cfg(unix)] +pub(crate) async fn restrict_permissions(f: &tokio::fs::File) { + use std::os::unix::fs::PermissionsExt; + let _ = f + .set_permissions(std::fs::Permissions::from_mode(0o600)) + .await; +} + +/// Path for a temp file alongside `path`, distinct per process. +/// +/// The previous fixed `.tmp` name meant two servers sharing a directory would +/// clobber each other's half-written snapshot, and made the target predictable +/// to anyone who could already write to that directory. Residual: this is not +/// unguessable, so it is a defence against collision rather than against an +/// attacker who already controls the data directory. +pub(crate) fn temp_sibling(path: &std::path::Path, tag: &str) -> PathBuf { + path.with_extension(format!("{tag}.{}.tmp", std::process::id())) +} + +// ── snapshot persistence ────────────────────────────────────────────────────── + +pub(crate) struct SnapshotConfig { + pub(crate) path: PathBuf, + pub(crate) last_save: AtomicI64, +} + +pub(crate) async fn save_snapshot(store: &KeyValueStore, cfg: &SnapshotConfig) { + let entries = store.snapshot(); + let count = entries.len(); + let tmp = temp_sibling(&cfg.path, "snap"); + match rmp_serde::to_vec(&entries) { + Err(e) => warn!("Snapshot serialize failed: {}", e), + Ok(bytes) => match write_private(&tmp, &bytes).await { + Err(e) => warn!("Snapshot write failed: {}", e), + Ok(()) => match tokio::fs::rename(&tmp, &cfg.path).await { + Err(e) => warn!("Snapshot rename failed: {}", e), + Ok(()) => { + sync_parent_dir(&cfg.path).await; + cfg.last_save.store(now_unix_secs(), Ordering::Relaxed); + info!("Snapshot saved: {} entries → {:?}", count, cfg.path); + } + }, + }, + } +} + +pub(crate) async fn load_snapshot(store: &KeyValueStore, path: &std::path::Path) -> bool { + match tokio::fs::read(path).await { + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + info!("No snapshot at {:?}, starting fresh", path); + false + } + Err(e) => { + warn!("Snapshot read failed: {}", e); + false + } + Ok(bytes) => match rmp_serde::from_slice::>(&bytes) { + Err(e) => { + warn!("Snapshot deserialize failed: {}", e); + false + } + Ok(entries) => { + let count = entries.len(); + store.restore(entries); + info!("Snapshot loaded: {} entries ← {:?}", count, path); + true + } + }, + } +} + +// ── AOF ─────────────────────────────────────────────────────────────────────── + +#[derive(Clone, Copy, PartialEq)] +pub(crate) enum AofSync { + Always, + EverySec, + No, +} + +pub(crate) struct AofWriter { + #[allow(dead_code)] + pub(crate) path: PathBuf, + pub(crate) file: tokio::sync::Mutex, + pub(crate) sync: AofSync, +} + +impl AofWriter { + pub(crate) async fn open(path: PathBuf, sync: AofSync) -> std::io::Result { + let mut opts = tokio::fs::OpenOptions::new(); + opts.create(true).append(true); + #[cfg(unix)] + opts.mode(0o600); + let file = opts.open(&path).await?; + // An AOF written by an earlier version is likely to be 0644 — tighten it + // on open, since `mode()` only applies to files this call creates. + #[cfg(unix)] + restrict_permissions(&file).await; + Ok(Self { + path, + file: tokio::sync::Mutex::new(file), + sync, + }) + } + + pub(crate) async fn append(&self, resp: &[u8]) { + let mut f = self.file.lock().await; + if f.write_all(resp).await.is_err() { + warn!("AOF write failed"); + return; + } + if self.sync == AofSync::Always { + // `flush()` alone only pushes tokio's buffer into a `write` syscall, + // which leaves the bytes in the page cache — surviving a process + // crash but not a power loss or kernel panic. `always` exists + // precisely to survive the latter, so it has to reach the device. + if let Err(e) = f.flush().await { + warn!("AOF flush failed: {}", e); + return; + } + if let Err(e) = f.sync_data().await { + warn!("AOF fsync failed: {}", e); + } + } + } + + /// Flush and fsync. Called on the `everysec` ticker and before shutdown. + /// + /// `sync_data` rather than `sync_all`: the AOF is append-only, so its + /// metadata beyond the length carries nothing worth an extra barrier. + pub(crate) async fn flush(&self) { + let mut f = self.file.lock().await; + if let Err(e) = f.flush().await { + warn!("AOF flush failed: {}", e); + return; + } + if let Err(e) = f.sync_data().await { + warn!("AOF fsync failed: {}", e); + } + } + + pub(crate) async fn truncate(&self) { + let f = self.file.lock().await; + match f.set_len(0).await { + // The truncation itself must be durable, or a crash can resurrect a + // log the snapshot has already subsumed and replay it on top. + Ok(()) => match f.sync_all().await { + Ok(()) => info!("AOF truncated after snapshot save"), + Err(e) => warn!("AOF truncate fsync failed: {}", e), + }, + Err(e) => warn!("AOF truncate failed: {}", e), + } + } +} + +pub(crate) async fn replay_aof(store: &KeyValueStore, path: &std::path::Path) -> usize { + let bytes = match tokio::fs::read(path).await { + Err(e) if e.kind() == ErrorKind::NotFound => return 0, + Err(e) => { + warn!("AOF read failed: {}", e); + return 0; + } + Ok(b) => b, + }; + let mut replayed = 0usize; + let mut offset = 0; + while offset < bytes.len() { + match Value::parse(&bytes[offset..]) { + Ok((value, consumed)) => { + offset += consumed; + // Writes are recorded via `on_write` in RESP3 Push form (`>N`); + // normalise to Array so Command::from_value can parse them. + let normalised = match value { + Value::Push(inner) => Value::Array(Some(inner)), + other => other, + }; + if let Ok(cmd) = Command::from_value(normalised) { + store.execute(cmd); + replayed += 1; + } + } + Err(e) if e.is_incomplete() => break, + Err(_) => { + warn!("AOF corrupted at offset {}, stopping replay", offset); + break; + } + } + } + if replayed > 0 { + info!("AOF replayed: {} commands ← {:?}", replayed, path); + } + replayed +} + +// ── Replication ─────────────────────────────────────────────────────────────── + +// ───────────────────────────────────────────────────────────────────────────── +// Durability: the guarantees `RECACHED_AOF_SYNC` and snapshot saving advertise. +// +// These assert reachability and effect rather than device-level durability — a +// unit test cannot pull the power. What they pin is that the fsync path is +// actually taken, that it does not corrupt or lose data, and that the pieces a +// crash-consistency argument depends on (temp file synced before rename, parent +// directory synced after) are wired up. +// ───────────────────────────────────────────────────────────────────────────── +#[cfg(test)] +mod durability_tests { + use super::*; + + fn scratch(name: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!( + "recached_dur_{}_{}_{}", + name, + std::process::id(), + next_conn_id() + )); + std::fs::create_dir_all(&dir).expect("create scratch dir"); + dir + } + + #[tokio::test] + async fn aof_always_survives_a_reopen_with_every_byte_intact() { + // `always` previously called flush(), which reaches the page cache and + // not the device. The observable part of the fix is that the fsync path + // runs and is still byte-exact. + let dir = scratch("aof_always"); + let path = dir.join("a.aof"); + let w = AofWriter::open(path.clone(), AofSync::Always) + .await + .unwrap(); + for i in 0..64 { + w.append(format!("*1\r\n${}\r\n{}\r\n", i.to_string().len(), i).as_bytes()) + .await; + } + drop(w); + + let on_disk = tokio::fs::read(&path).await.unwrap(); + let expected: Vec = (0..64) + .flat_map(|i| format!("*1\r\n${}\r\n{}\r\n", i.to_string().len(), i).into_bytes()) + .collect(); + assert_eq!(on_disk, expected, "fsync must not disturb the byte stream"); + + let _ = tokio::fs::remove_dir_all(&dir).await; + } + + #[tokio::test] + async fn aof_everysec_flush_is_idempotent_and_lossless() { + // The everysec ticker calls flush() on a cadence, including when nothing + // has been appended since the last tick. + let dir = scratch("aof_everysec"); + let path = dir.join("b.aof"); + let w = AofWriter::open(path.clone(), AofSync::EverySec) + .await + .unwrap(); + w.append(b"*1\r\n$4\r\nPING\r\n").await; + w.flush().await; + w.flush().await; // nothing new to sync + assert_eq!( + tokio::fs::read(&path).await.unwrap(), + b"*1\r\n$4\r\nPING\r\n" + ); + let _ = tokio::fs::remove_dir_all(&dir).await; + } + + #[tokio::test] + async fn truncating_the_aof_leaves_it_empty_and_reusable() { + // Truncation follows a snapshot, and is now fsynced so a crash cannot + // resurrect a log the snapshot already subsumed. It must also leave the + // handle writable — the server keeps appending to it afterwards. + let dir = scratch("aof_trunc"); + let path = dir.join("c.aof"); + let w = AofWriter::open(path.clone(), AofSync::No).await.unwrap(); + w.append(b"*1\r\n$4\r\nPING\r\n").await; + w.truncate().await; + assert_eq!(tokio::fs::metadata(&path).await.unwrap().len(), 0); + + w.append(b"*1\r\n$4\r\nECHO\r\n").await; + w.flush().await; + assert_eq!( + tokio::fs::read(&path).await.unwrap(), + b"*1\r\n$4\r\nECHO\r\n", + "the writer must still be usable after truncation" + ); + let _ = tokio::fs::remove_dir_all(&dir).await; + } + + #[tokio::test] + async fn a_snapshot_lands_atomically_and_leaves_no_temp_file() { + // The temp file is fsynced, renamed over the target, and then the + // directory is fsynced. A leftover temp file would mean the rename never + // happened, which is the failure this sequence exists to prevent. + let dir = scratch("snap"); + let path = dir.join("dump.rdb"); + let store = KeyValueStore::new(); + for i in 0..32 { + store.execute(Command::Set( + format!("k{i}"), + format!("v{i}").into_bytes(), + Default::default(), + )); + } + let cfg = SnapshotConfig { + path: path.clone(), + last_save: AtomicI64::new(0), + }; + save_snapshot(&store, &cfg).await; + + assert!(path.exists(), "snapshot must exist after save"); + assert!( + cfg.last_save.load(Ordering::Relaxed) > 0, + "a completed save must advance LASTSAVE" + ); + + let leftovers: Vec<_> = std::fs::read_dir(&dir) + .unwrap() + .filter_map(Result::ok) + .map(|e| e.file_name().to_string_lossy().into_owned()) + .filter(|n| n.ends_with(".tmp")) + .collect(); + assert!( + leftovers.is_empty(), + "temp files left behind: {leftovers:?}" + ); + + // And the bytes are a snapshot we can actually read back. + let restored = KeyValueStore::new(); + assert!(load_snapshot(&restored, &path).await); + assert_eq!( + restored.execute(Command::Get("k7".into())), + Value::BulkString(Some(b"v7".to_vec())) + ); + + let _ = tokio::fs::remove_dir_all(&dir).await; + } + + #[tokio::test] + async fn syncing_a_parent_directory_tolerates_odd_paths() { + // Best-effort by contract: a path with no parent, or one that does not + // exist, must warn rather than panic or hang. The snapshot path is + // frequently relative ("recached.rdb"), which has an empty parent. + sync_parent_dir(std::path::Path::new("recached.rdb")).await; + sync_parent_dir(std::path::Path::new("/")).await; + sync_parent_dir(std::path::Path::new("/nonexistent-recached-dir/x.rdb")).await; + } + + #[test] + fn a_sync_token_cannot_grant_an_over_long_pattern() { + // Token patterns reach glob_match without passing through the command + // parser, so the cap has to be repeated there. These are matched once + // per key per write — the most expensive place a pattern can sit. + use base64::Engine as _; + use hmac::{Hmac, Mac}; + use sha2::Sha256; + + let secret = "s3cret"; + let engine = base64::engine::general_purpose::URL_SAFE_NO_PAD; + let mint = |payload: &str| { + let p = engine.encode(payload); + let mut mac = Hmac::::new_from_slice(secret.as_bytes()).unwrap(); + mac.update(p.as_bytes()); + format!("{}.{}", p, engine.encode(mac.finalize().into_bytes())) + }; + + let long = "a".repeat(core_engine::store::MAX_PATTERN_BYTES + 1); + let err = verify_sync_token(secret, &mint(&long)) + .expect_err("an over-long granted pattern must be refused"); + assert_eq!(err, "token grants an over-long pattern"); + + // One over-long pattern in an otherwise fine list is still refused. + assert!(verify_sync_token(secret, &mint(&format!("cart:*,{long}"))).is_err()); + + // A pattern exactly at the cap is still honoured. + let at_cap = "a".repeat(core_engine::store::MAX_PATTERN_BYTES); + assert_eq!( + verify_sync_token(secret, &mint(&at_cap)), + Ok(vec![at_cap.clone()]) + ); + assert_eq!( + verify_sync_token(secret, &mint("cart:42:*,user:1:*")), + Ok(vec!["cart:42:*".to_string(), "user:1:*".to_string()]) + ); + } +} + +// ── Expiry propagation ──────────────────────────────────────────────────────── diff --git a/server-native/src/propagation.rs b/server-native/src/propagation.rs new file mode 100644 index 0000000..6bc1eed --- /dev/null +++ b/server-native/src/propagation.rs @@ -0,0 +1,1214 @@ +//! Turning an executed command into the frame that reaches the AOF, replicas +//! and browser sync clients — and the watch notifications that go with it. + +use crate::*; + +pub(crate) fn is_write_command(cmd: &Command) -> bool { + if let Command::Dedup(_, _, inner) = cmd { + return is_write_command(inner); + } + matches!( + cmd, + Command::Set(..) + | Command::ESet(..) + | Command::Del(..) + | Command::Unlink(..) + | Command::Append(..) + | Command::GetSet(..) + | Command::MSet(..) + | Command::SetNx(..) + | Command::SetEx(..) + | Command::PSetEx(..) + | Command::Incr(..) + | Command::Decr(..) + | Command::IncrBy(..) + | Command::DecrBy(..) + | Command::Expire(..) + | Command::PExpire(..) + | Command::ExpireAt(..) + | Command::PExpireAt(..) + | Command::Persist(..) + | Command::FlushDb + | Command::Rename(..) + | Command::HSet(..) + | Command::HDel(..) + | Command::HIncrBy(..) + | Command::HIncrByFloat(..) + | Command::HSetNx(..) + | Command::LPush(..) + | Command::RPush(..) + | Command::LPushX(..) + | Command::RPushX(..) + | Command::LPop(..) + | Command::RPop(..) + | Command::LSet(..) + | Command::LRem(..) + | Command::LTrim(..) + | Command::SAdd(..) + | Command::SRem(..) + | Command::SInterStore(..) + | Command::SUnionStore(..) + | Command::SDiffStore(..) + | Command::SPop(..) + | Command::SMove(..) + | Command::ZAdd(..) + | Command::ZRem(..) + | Command::ZIncrBy(..) + | Command::RlSet(..) + | Command::RlCheck(..) + | Command::JSet(..) + | Command::JMerge(..) + ) +} + +// ── Save conditions ─────────────────────────────────────────────────────────── + +/// Extract the key(s) that `cmd` writes to, without inspecting the response. +/// Used together with `broadcast_for()` — only call this when `broadcast_for` +/// already confirmed a mutation occurred. +pub(crate) fn primary_keys(cmd: &Command) -> Vec { + match cmd { + Command::ESet(k, _) + | Command::Set(k, _, _) + | Command::Append(k, _) + | Command::GetSet(k, _) + | Command::SetNx(k, _) + | Command::SetEx(k, _, _) + | Command::PSetEx(k, _, _) + | Command::Incr(k) + | Command::Decr(k) + | Command::IncrBy(k, _) + | Command::DecrBy(k, _) + | Command::Expire(k, _) + | Command::PExpire(k, _) + | Command::ExpireAt(k, _) + | Command::PExpireAt(k, _) + | Command::Persist(k) + | Command::HSet(k, _) + | Command::HDel(k, _) + | Command::HSetNx(k, _, _) + | Command::HIncrBy(k, _, _) + | Command::HIncrByFloat(k, _, _) + | Command::LPush(k, _) + | Command::RPush(k, _) + | Command::LPushX(k, _) + | Command::RPushX(k, _) + | Command::LPop(k, _) + | Command::RPop(k, _) + | Command::LSet(k, _, _) + | Command::LRem(k, _, _) + | Command::LTrim(k, _, _) + | Command::SAdd(k, _) + | Command::SRem(k, _) + | Command::SPop(k, _) + | Command::SInterStore(k, _) + | Command::SUnionStore(k, _) + | Command::SDiffStore(k, _) + | Command::ZAdd(k, _, _) + | Command::ZRem(k, _) + | Command::ZIncrBy(k, _, _) + | Command::RlSet(k, _, _) + | Command::JSet(k, _, _) + | Command::JMerge(k, _) => vec![k.clone()], + Command::Del(keys) | Command::Unlink(keys) => keys.clone(), + Command::MSet(pairs) => pairs.iter().map(|(k, _)| k.clone()).collect(), + Command::Rename(src, dst) | Command::SMove(src, dst, _) => { + vec![src.clone(), dst.clone()] + } + _ => vec![], + } +} + +pub(crate) fn encode_keychange(key: &str, value: &Value) -> Vec { + Value::Array(Some(vec![ + Value::BulkString(Some(b"keychange".to_vec())), + Value::BulkString(Some(key.as_bytes().to_vec())), + value.clone(), + ])) + .serialize() +} + +/// Push keychange notifications for a *confirmed* mutation. Callers must have +/// already established that `cmd` mutated the store (via `broadcast_for`). +pub(crate) async fn notify_watchers( + registry: &WatchRegistry, + cmd: &Command, + store: &KeyValueStore, +) { + if registry.is_empty() { + return; + } + let keys = primary_keys(cmd); + if keys.is_empty() { + return; + } + // Fetch current values from DashMap *before* acquiring the registry lock + // to avoid holding two locks simultaneously. + let key_values: Vec<(String, Value)> = keys + .iter() + .map(|k| (k.clone(), store.get_current(k))) + .collect(); + if registry.watched_keys.load(Ordering::Relaxed) > 0 { + let mut reg = registry.map.lock().await; + for (key, value) in &key_values { + if let Some(subs) = reg.get_mut(key) { + subs.retain(|(_, tx)| tx.send((key.clone(), value.clone())).is_ok()); + if subs.is_empty() { + reg.remove(key); + } + } + } + registry.sync_len(®); + } + // Live queries: any registered glob pattern matching a touched key gets + // the same keychange notification. + if registry.watched_patterns.load(Ordering::Relaxed) > 0 { + let mut pats = registry.patterns.lock().await; + let mut emptied = false; + for (pattern, subs) in pats.iter_mut() { + for (key, value) in &key_values { + if core_engine::store::glob_match(pattern, key) { + subs.retain(|(_, tx)| tx.send((key.clone(), value.clone())).is_ok()); + } + } + emptied |= subs.is_empty(); + } + if emptied { + pats.retain(|_, subs| !subs.is_empty()); + } + registry.sync_patterns_len(&pats); + } +} + +/// Announce a `FLUSHDB` to live queries. +/// +/// Emitting a keychange per deleted key would mean one frame per key in the +/// keyspace — potentially millions — for a single command. Instead each +/// registered pattern receives one sentinel, delivered as a keychange whose key +/// is the pattern and whose value is nil. Subscribers treat it as "every key +/// matching this pattern is gone", which is exactly what happened, at O(patterns) +/// instead of O(keys). +/// +/// Explicitly `WATCH`ed keys are notified individually — that set is bounded by +/// the connection limit and callers expect per-key precision there. +pub(crate) async fn notify_flushdb(registry: &WatchRegistry, watched_before: Vec) { + if registry.watched_keys.load(Ordering::Relaxed) > 0 && !watched_before.is_empty() { + let mut reg = registry.map.lock().await; + for key in &watched_before { + if let Some(subs) = reg.get_mut(key) { + subs.retain(|(_, tx)| tx.send((key.clone(), Value::BulkString(None))).is_ok()); + } + } + registry.sync_len(®); + } + if registry.watched_patterns.load(Ordering::Relaxed) > 0 { + let mut pats = registry.patterns.lock().await; + let mut emptied = false; + for (pattern, subs) in pats.iter_mut() { + let sentinel = pattern.clone(); + subs.retain(|(_, tx)| tx.send((sentinel.clone(), Value::BulkString(None))).is_ok()); + emptied |= subs.is_empty(); + } + if emptied { + pats.retain(|_, subs| !subs.is_empty()); + } + registry.sync_patterns_len(&pats); + } +} + +/// Post-write fan-out shared by the TCP and WS command paths: WebSocket sync +/// broadcast, AOF/replication log, and watch notifications. Structured so that +/// with no WS clients, no replicas, no AOF, and no watched keys — the common +/// standalone-server case — a write costs zero locks and zero allocations here. +pub(crate) async fn apply_write_effects( + cmd: &Command, + response: &Value, + tx: &broadcast::Sender, + origin: u64, + state: &ServerState, + watch_registry: &WatchRegistry, + store: &KeyValueStore, +) { + let has_ws = tx.receiver_count() > 0; + let needs_log = state.needs_write_log(); + let has_watch = !watch_registry.is_empty(); + if !has_ws && !needs_log && !has_watch { + return; + } + // Read once, after the early-out above, so a standalone server with nothing + // listening still pays no clock read per write. + let Some(msg) = broadcast_for(cmd, response, now_unix_ms()) else { + return; + }; + if needs_log { + state.on_write(&msg).await; + } + if has_watch { + if matches!(cmd, Command::FlushDb) { + // primary_keys() is empty for FLUSHDB, so the generic notifier has + // nothing to announce — subscribers would silently miss the wipe. + let watched: Vec = { + let reg = watch_registry.map.lock().await; + reg.keys().cloned().collect() + }; + notify_flushdb(watch_registry, watched).await; + } else { + notify_watchers(watch_registry, cmd, store).await; + } + } + if has_ws { + let _ = tx.send(Arc::new(SyncPush { + origin, + keys: primary_keys(cmd), + resp: msg, + })); + } +} + +/// Encodes a list of string parts as a RESP3 Push frame for WebSocket fan-out. +/// Uses `>` prefix so clients can distinguish server-initiated pushes from command responses. +/// Build a RESP3 Push frame from raw byte arguments. +/// +/// Bytes rather than `&str` because these frames carry stored values, which may +/// be arbitrary binary. Building them as a `String` would have required a lossy +/// conversion — silently corrupting the replicated, AOF-logged and +/// browser-synced copy of a value the store itself holds faithfully. +pub(crate) fn resp_push(parts: &[&[u8]]) -> Vec { + let mut out = format!(">{}\r\n", parts.len()).into_bytes(); + for part in parts { + out.extend_from_slice(format!("${}\r\n", part.len()).as_bytes()); + out.extend_from_slice(part); + out.extend_from_slice(b"\r\n"); + } + out +} + +/// Returns the RESP-encoded mutation to broadcast to WebSocket peers, or `None` +/// if the command mutated nothing (read-only or conditional-and-failed). +/// +/// `now_ms` is the propagation timestamp, and every *relative* TTL is rewritten +/// against it into an absolute `PXAT`/`PEXPIREAT` deadline. +/// +/// That rewrite is load-bearing, because this one buffer is what the AOF, the +/// replication log and the browser sync fan-out all receive. Propagating the +/// relative form meant each of them re-based the TTL onto *its own* clock at +/// *its own* arrival time, so a key's lifetime silently restarted on every hop: +/// +/// - **AOF** — replay happens at startup, so a key written with `EX 5` and +/// replayed an hour later came back alive with a fresh 5 seconds. A revoked +/// session, a distributed lock or an idempotency key that had long since +/// expired was resurrected by a restart. +/// - **Replicas** — a replica's copy expired later than the primary's by the +/// replication delay, and the gap reopened on every re-send. +/// - **Browsers** — the sync socket delivers on connect *and* on outbox replay, +/// so a reconnecting tab reset the TTL of every key it received. +/// +/// An absolute deadline is idempotent under replay: applying it once or a +/// thousand times, now or after an hour of downtime, yields the same instant. +/// A deadline already in the past is not a special case — the store treats such +/// an entry as expired on read and the sweeper reaps it, which is precisely the +/// "it should already be gone" behaviour that was missing. +/// +/// The deadline is computed from `now_ms` rather than read back out of the +/// store: the store's own expiry was computed microseconds earlier from the +/// same clock, and re-reading it would cost a lookup per write and still race +/// another thread overwriting the key. This is also what Redis does — it +/// rewrites relative expiries to absolute ones at propagation time. +pub(crate) fn broadcast_for(cmd: &Command, response: &Value, now_ms: u64) -> Option> { + match cmd { + // Replays as SET: a replica has no connection to scope the lifetime to, + // and the owning server broadcasts the DEL when the connection closes. + Command::ESet(k, v) => Some(resp_push(&[b"SET", k.as_bytes(), v.as_slice()])), + Command::Set(k, v, opts) => { + // Without GET: nil response means NX/XX condition failed — don't broadcast. + // With GET: nil means key didn't exist before, but SET still happened. + let set_happened = opts.get || !matches!(response, Value::BulkString(None)); + if !set_happened { + return None; + } + match &opts.expiry { + None => Some(resp_push(&[b"SET", k.as_bytes(), v.as_slice()])), + // Relative → absolute: see the note on `broadcast_for`. + Some(SetExpiry::Ex(s)) => { + let pxat = now_ms.saturating_add(s.saturating_mul(1000)).to_string(); + Some(resp_push(&[ + b"SET", + k.as_bytes(), + v.as_slice(), + b"PXAT", + pxat.as_bytes(), + ])) + } + Some(SetExpiry::Px(ms)) => { + let pxat = now_ms.saturating_add(*ms).to_string(); + Some(resp_push(&[ + b"SET", + k.as_bytes(), + v.as_slice(), + b"PXAT", + pxat.as_bytes(), + ])) + } + Some(SetExpiry::Exat(ts)) => { + let pxat = ts.saturating_mul(1000).to_string(); + Some(resp_push(&[ + b"SET", + k.as_bytes(), + v.as_slice(), + b"PXAT", + pxat.as_bytes(), + ])) + } + Some(SetExpiry::Pxat(ts)) => { + let ts_s = ts.to_string(); + Some(resp_push(&[ + b"SET", + k.as_bytes(), + v.as_slice(), + b"PXAT", + ts_s.as_bytes(), + ])) + } + Some(SetExpiry::KeepTtl) => { + Some(resp_push(&[b"SET", k.as_bytes(), v.as_slice(), b"KEEPTTL"])) + } + } + } + Command::Del(keys) | Command::Unlink(keys) => { + let mut parts: Vec<&[u8]> = vec![b"DEL"]; + let key_refs: Vec<&[u8]> = keys.iter().map(|s| s.as_bytes()).collect(); + parts.extend_from_slice(&key_refs); + Some(resp_push(&parts)) + } + Command::MSet(pairs) => { + let mut parts: Vec<&[u8]> = vec![b"MSET"]; + let flat: Vec> = pairs + .iter() + .flat_map(|(k, v)| [k.as_bytes().to_vec(), v.clone()]) + .collect(); + let flat_refs: Vec<&[u8]> = flat.iter().map(|s| s.as_slice()).collect(); + parts.extend_from_slice(&flat_refs); + Some(resp_push(&parts)) + } + Command::SetNx(k, v) => match response { + Value::Integer(1) => Some(resp_push(&[b"SET", k.as_bytes(), v.as_slice()])), + _ => None, + }, + Command::SetEx(k, secs, v) => { + let pxat = now_ms.saturating_add(secs.saturating_mul(1000)).to_string(); + Some(resp_push(&[ + b"SET", + k.as_bytes(), + v.as_slice(), + b"PXAT", + pxat.as_bytes(), + ])) + } + Command::PSetEx(k, ms, v) => { + let pxat = now_ms.saturating_add(*ms).to_string(); + Some(resp_push(&[ + b"SET", + k.as_bytes(), + v.as_slice(), + b"PXAT", + pxat.as_bytes(), + ])) + } + Command::Append(k, v) => match response { + Value::Integer(_) => Some(resp_push(&[b"APPEND", k.as_bytes(), v.as_slice()])), + _ => None, + }, + // GETSET clears any TTL, as in Redis, so a bare SET is the faithful + // replay — unlike the counters below. + Command::GetSet(k, v) => Some(resp_push(&[b"SET", k.as_bytes(), v.as_slice()])), + // Counters replay as `SET KEEPTTL`. + // + // A counter is propagated by value rather than as `INCR`, so that a + // replica that missed a frame converges on the primary's number instead + // of compounding its own. But a bare `SET` also *clears* the TTL, and + // `INCR` in Redis leaves it untouched — so the single most common + // expiring-counter idiom, `INCR key` + `EXPIRE key window`, replayed as + // a key with no expiry at all. The rate-limit bucket, the per-minute + // quota and the retry counter all became permanent on the replica, in + // the AOF and in every synced browser, and the next window never reset + // because the key it keyed on never went away. `KEEPTTL` keeps the + // by-value convergence while leaving the deadline where the primary + // has it. + Command::Incr(k) | Command::Decr(k) | Command::IncrBy(k, _) | Command::DecrBy(k, _) => { + match response { + Value::Integer(n) => { + let s = n.to_string(); + Some(resp_push(&[b"SET", k.as_bytes(), s.as_bytes(), b"KEEPTTL"])) + } + _ => None, + } + } + // Relative → absolute: see the note on `broadcast_for`. + Command::Expire(k, secs) => match response { + Value::Integer(1) => { + let ts = now_ms.saturating_add(secs.saturating_mul(1000)).to_string(); + Some(resp_push(&[b"PEXPIREAT", k.as_bytes(), ts.as_bytes()])) + } + _ => None, + }, + Command::PExpire(k, ms) => match response { + Value::Integer(1) => { + let ts = now_ms.saturating_add(*ms).to_string(); + Some(resp_push(&[b"PEXPIREAT", k.as_bytes(), ts.as_bytes()])) + } + _ => None, + }, + Command::ExpireAt(k, ts) => match response { + Value::Integer(1) => { + let ts_ms = ts.saturating_mul(1000).to_string(); + Some(resp_push(&[b"PEXPIREAT", k.as_bytes(), ts_ms.as_bytes()])) + } + _ => None, + }, + Command::PExpireAt(k, ts) => match response { + Value::Integer(1) => { + let ts_s = ts.to_string(); + Some(resp_push(&[b"PEXPIREAT", k.as_bytes(), ts_s.as_bytes()])) + } + _ => None, + }, + Command::Persist(k) => match response { + Value::Integer(1) => Some(resp_push(&[b"PERSIST", k.as_bytes()])), + _ => None, + }, + Command::FlushDb => Some(resp_push(&[b"FLUSHDB"])), + Command::Rename(src, dst) => match response { + Value::Error(_) => None, + _ => Some(resp_push(&[b"RENAME", src.as_bytes(), dst.as_bytes()])), + }, + + // ── Hash ───────────────────────────────────────────────────────────── + Command::HSet(k, pairs) => { + let mut parts: Vec> = vec![b"HSET".to_vec(), k.as_bytes().to_vec()]; + for (f, v) in pairs { + parts.push(f.as_bytes().to_vec()); + parts.push(v.clone()); + } + let refs: Vec<&[u8]> = parts.iter().map(|s| s.as_slice()).collect(); + Some(resp_push(&refs)) + } + Command::HDel(k, fields) => match response { + Value::Integer(n) if *n > 0 => { + let mut parts: Vec<&[u8]> = vec![b"HDEL", k.as_bytes()]; + let field_refs: Vec<&[u8]> = fields.iter().map(|s| s.as_bytes()).collect(); + parts.extend_from_slice(&field_refs); + Some(resp_push(&parts)) + } + _ => None, + }, + Command::HIncrBy(k, f, _) => match response { + Value::Integer(n) => { + let s = n.to_string(); + Some(resp_push(&[ + b"HSET", + k.as_bytes(), + f.as_bytes(), + s.as_bytes(), + ])) + } + _ => None, + }, + Command::HIncrByFloat(k, f, _) => match response { + Value::BulkString(Some(data)) => { + let s = String::from_utf8_lossy(data); + Some(resp_push(&[ + b"HSET", + k.as_bytes(), + f.as_bytes(), + s.as_bytes(), + ])) + } + _ => None, + }, + Command::HSetNx(k, f, v) => match response { + Value::Integer(1) => Some(resp_push(&[ + b"HSET", + k.as_bytes(), + f.as_bytes(), + v.as_slice(), + ])), + _ => None, + }, + + // ── List ───────────────────────────────────────────────────────────── + Command::LPush(k, vals) | Command::RPush(k, vals) => { + let cmd_name = if matches!(cmd, Command::LPush(_, _)) { + "LPUSH" + } else { + "RPUSH" + }; + let mut parts: Vec<&[u8]> = vec![cmd_name.as_bytes(), k.as_bytes()]; + let val_refs: Vec<&[u8]> = vals.iter().map(|v| v.as_slice()).collect(); + parts.extend_from_slice(&val_refs); + Some(resp_push(&parts)) + } + Command::LPushX(k, vals) | Command::RPushX(k, vals) => match response { + Value::Integer(n) if *n > 0 => { + let cmd_name = if matches!(cmd, Command::LPushX(_, _)) { + "LPUSH" + } else { + "RPUSH" + }; + let mut parts: Vec<&[u8]> = vec![cmd_name.as_bytes(), k.as_bytes()]; + let val_refs: Vec<&[u8]> = vals.iter().map(|v| v.as_slice()).collect(); + parts.extend_from_slice(&val_refs); + Some(resp_push(&parts)) + } + _ => None, + }, + Command::LPop(k, count) => match response { + Value::BulkString(None) => None, + Value::Array(Some(items)) if items.is_empty() => None, + _ => { + let n = count.map(|c| c.to_string()); + match &n { + Some(ns) => Some(resp_push(&[b"LPOP", k.as_bytes(), ns.as_bytes()])), + None => Some(resp_push(&[b"LPOP", k.as_bytes()])), + } + } + }, + Command::RPop(k, count) => match response { + Value::BulkString(None) => None, + Value::Array(Some(items)) if items.is_empty() => None, + _ => { + let n = count.map(|c| c.to_string()); + match &n { + Some(ns) => Some(resp_push(&[b"RPOP", k.as_bytes(), ns.as_bytes()])), + None => Some(resp_push(&[b"RPOP", k.as_bytes()])), + } + } + }, + Command::LSet(k, idx, v) => match response { + Value::SimpleString(_) => { + let idx_s = idx.to_string(); + Some(resp_push(&[ + b"LSET", + k.as_bytes(), + idx_s.as_bytes(), + v.as_slice(), + ])) + } + _ => None, + }, + Command::LRem(k, count, elem) => match response { + Value::Integer(n) if *n > 0 => { + let count_s = count.to_string(); + Some(resp_push(&[ + b"LREM", + k.as_bytes(), + count_s.as_bytes(), + elem.as_slice(), + ])) + } + _ => None, + }, + Command::LTrim(k, start, stop) => { + let start_s = start.to_string(); + let stop_s = stop.to_string(); + Some(resp_push(&[ + b"LTRIM", + k.as_bytes(), + start_s.as_bytes(), + stop_s.as_bytes(), + ])) + } + + // ── Set ─────────────────────────────────────────────────────────────── + Command::SAdd(k, members) => match response { + Value::Integer(n) if *n > 0 => { + let mut parts: Vec<&[u8]> = vec![b"SADD", k.as_bytes()]; + let m_refs: Vec<&[u8]> = members.iter().map(|s| s.as_bytes()).collect(); + parts.extend_from_slice(&m_refs); + Some(resp_push(&parts)) + } + _ => None, + }, + Command::SRem(k, members) => match response { + Value::Integer(n) if *n > 0 => { + let mut parts: Vec<&[u8]> = vec![b"SREM", k.as_bytes()]; + let m_refs: Vec<&[u8]> = members.iter().map(|s| s.as_bytes()).collect(); + parts.extend_from_slice(&m_refs); + Some(resp_push(&parts)) + } + _ => None, + }, + Command::SPop(k, count) => { + let popped: Vec = match response { + Value::BulkString(Some(data)) => { + vec![String::from_utf8_lossy(data).into_owned()] + } + Value::Array(Some(items)) => items + .iter() + .filter_map(|v| { + if let Value::BulkString(Some(d)) = v { + Some(String::from_utf8_lossy(d).into_owned()) + } else { + None + } + }) + .collect(), + _ => vec![], + }; + if popped.is_empty() { + let _ = count; + None + } else { + let mut parts: Vec<&[u8]> = vec![b"SREM", k.as_bytes()]; + let m_refs: Vec<&[u8]> = popped.iter().map(|s| s.as_bytes()).collect(); + parts.extend_from_slice(&m_refs); + Some(resp_push(&parts)) + } + } + Command::SMove(src, dst, member) => match response { + Value::Integer(1) => Some(resp_push(&[ + b"SMOVE", + src.as_bytes(), + dst.as_bytes(), + member.as_bytes(), + ])), + _ => None, + }, + Command::SInterStore(dst, keys) => { + let mut parts: Vec<&[u8]> = vec![b"SINTERSTORE", dst.as_bytes()]; + let k_refs: Vec<&[u8]> = keys.iter().map(|s| s.as_bytes()).collect(); + parts.extend_from_slice(&k_refs); + Some(resp_push(&parts)) + } + Command::SUnionStore(dst, keys) => { + let mut parts: Vec<&[u8]> = vec![b"SUNIONSTORE", dst.as_bytes()]; + let k_refs: Vec<&[u8]> = keys.iter().map(|s| s.as_bytes()).collect(); + parts.extend_from_slice(&k_refs); + Some(resp_push(&parts)) + } + Command::SDiffStore(dst, keys) => { + let mut parts: Vec<&[u8]> = vec![b"SDIFFSTORE", dst.as_bytes()]; + let k_refs: Vec<&[u8]> = keys.iter().map(|s| s.as_bytes()).collect(); + parts.extend_from_slice(&k_refs); + Some(resp_push(&parts)) + } + + // ── Sorted Set ──────────────────────────────────────────────────────── + Command::ZAdd(k, opts, pairs) => { + let mut parts: Vec = vec!["ZADD".into(), k.clone()]; + if let Some(cond) = &opts.condition { + parts.push(match cond { + ZAddCondition::Nx => "NX".into(), + ZAddCondition::Xx => "XX".into(), + }); + } + if opts.ch { + parts.push("CH".into()); + } + if opts.incr { + parts.push("INCR".into()); + } + for (score, member) in pairs { + parts.push(format_f64_score(*score)); + parts.push(member.clone()); + } + let refs: Vec<&[u8]> = parts.iter().map(|s| s.as_bytes()).collect(); + Some(resp_push(&refs)) + } + Command::ZRem(k, members) => match response { + Value::Integer(n) if *n > 0 => { + let mut parts: Vec<&[u8]> = vec![b"ZREM", k.as_bytes()]; + let m_refs: Vec<&[u8]> = members.iter().map(|s| s.as_bytes()).collect(); + parts.extend_from_slice(&m_refs); + Some(resp_push(&parts)) + } + _ => None, + }, + Command::ZIncrBy(k, delta, member) => { + let delta_s = format_f64_score(*delta); + Some(resp_push(&[ + b"ZINCRBY", + k.as_bytes(), + delta_s.as_bytes(), + member.as_bytes(), + ])) + } + + // ── JSON ───────────────────────────────────────────────────────────── + // Replayable as-is on replicas, AOF, and browser stores. Only + // successful writes replicate (errors reply -ERR, not +OK). + Command::JSet(k, path, value) => match response { + Value::SimpleString(_) => Some(resp_push(&[ + b"JSET", + k.as_bytes(), + path.as_bytes(), + value.as_bytes(), + ])), + _ => None, + }, + Command::JMerge(k, patch) => match response { + Value::SimpleString(_) => Some(resp_push(&[b"JMERGE", k.as_bytes(), patch.as_bytes()])), + _ => None, + }, + + // ── Rate limiting ──────────────────────────────────────────────────── + // RLSET replicates so limiter *config* survives AOF replay / reaches + // replicas. RLCHECK is deliberately not replicated: attempt state is + // transient and high-frequency — streaming every check would flood the + // AOF and the sync fan-out for state that expires within one window. + Command::RlSet(k, limit, window_secs) => { + let limit_s = limit.to_string(); + let window_s = window_secs.to_string(); + Some(resp_push(&[ + b"RLSET", + k.as_bytes(), + limit_s.as_bytes(), + window_s.as_bytes(), + ])) + } + + // Pub/Sub and transactions carry no store state — no broadcast needed. + _ => None, + } +} + +pub(crate) fn format_f64_score(s: f64) -> String { + if s == f64::INFINITY { + "inf".into() + } else if s == f64::NEG_INFINITY { + "-inf".into() + } else if s.fract() == 0.0 && s.abs() < 1e15 { + format!("{}", s as i64) + } else { + format!("{}", s) + } +} + +/// `broadcast_for` emits one buffer that the AOF, the replication log and the +/// browser sync fan-out all consume. It used to propagate *relative* TTLs +/// (`PX 5000`), so each consumer re-based the deadline onto its own clock at its +/// own arrival time and a key's lifetime silently restarted on every hop — most +/// visibly at AOF replay, where a long-dead key came back with a full fresh TTL. +/// +/// These tests pin the replacement contract: relative in, absolute out. +#[cfg(test)] +mod expiry_propagation_tests { + use super::*; + use core_engine::cmd::SetOptions; + use core_engine::store::KeyValueStore; + + fn tmp_path(name: &str) -> PathBuf { + std::env::temp_dir().join(format!("recached_test_{name}_{}", std::process::id())) + } + + /// Decode a broadcast frame into its argument strings. + fn parts(frame: &[u8]) -> Vec { + let (v, n) = Value::parse(frame).expect("frame must parse"); + assert_eq!(n, frame.len(), "frame must be exactly one value"); + let items = match v { + Value::Push(items) | Value::Array(Some(items)) => items, + other => panic!("expected an aggregate, got {other:?}"), + }; + items + .into_iter() + .map(|i| match i { + Value::BulkString(Some(b)) => String::from_utf8_lossy(&b).into_owned(), + other => panic!("expected a bulk string, got {other:?}"), + }) + .collect() + } + + fn frame_of(cmd: &Command, now_ms: u64) -> Vec { + let f = broadcast_for(cmd, &Value::SimpleString("OK".into()), now_ms) + .expect("command must propagate"); + parts(&f) + } + + const NOW: u64 = 1_700_000_000_000; + + #[test] + fn relative_set_expiries_propagate_as_an_absolute_deadline() { + let set = |exp| { + Command::Set( + "k".into(), + "v".into(), + SetOptions { + expiry: Some(exp), + ..Default::default() + }, + ) + }; + // EX seconds and PX milliseconds both land on the same instant. + for (cmd, want) in [ + (set(SetExpiry::Ex(5)), NOW + 5_000), + (set(SetExpiry::Px(1_500)), NOW + 1_500), + (Command::SetEx("k".into(), 5, "v".into()), NOW + 5_000), + (Command::PSetEx("k".into(), 1_500, "v".into()), NOW + 1_500), + ] { + let p = frame_of(&cmd, NOW); + assert_eq!(p[0], "SET", "{p:?}"); + assert_eq!( + p[3], "PXAT", + "a relative TTL must not reach the log as PX: {p:?}" + ); + assert_eq!(p[4], want.to_string(), "{p:?}"); + } + } + + #[test] + fn relative_expire_commands_propagate_as_an_absolute_deadline() { + for (cmd, want) in [ + (Command::Expire("k".into(), 30), NOW + 30_000), + (Command::PExpire("k".into(), 250), NOW + 250), + ] { + let f = broadcast_for(&cmd, &Value::Integer(1), NOW).expect("must propagate"); + let p = parts(&f); + assert_eq!(p[0], "PEXPIREAT", "{p:?}"); + assert_eq!(p[2], want.to_string(), "{p:?}"); + } + } + + #[test] + fn absolute_expiries_are_still_passed_through_unchanged() { + // These arms were already correct; the rewrite must not double-convert + // them by adding `now` to a stamp that is already absolute. + let set = |exp| { + Command::Set( + "k".into(), + "v".into(), + SetOptions { + expiry: Some(exp), + ..Default::default() + }, + ) + }; + let p = frame_of(&set(SetExpiry::Pxat(999)), NOW); + assert_eq!((p[3].as_str(), p[4].as_str()), ("PXAT", "999"), "{p:?}"); + let p = frame_of(&set(SetExpiry::Exat(999)), NOW); + assert_eq!((p[3].as_str(), p[4].as_str()), ("PXAT", "999000"), "{p:?}"); + + let f = broadcast_for( + &Command::PExpireAt("k".into(), 999), + &Value::Integer(1), + NOW, + ) + .expect("must propagate"); + assert_eq!(parts(&f)[2], "999"); + let f = broadcast_for(&Command::ExpireAt("k".into(), 999), &Value::Integer(1), NOW) + .expect("must propagate"); + assert_eq!(parts(&f)[2], "999000"); + } + + #[test] + fn a_write_without_an_expiry_still_carries_none() { + // A plain SET clears any TTL, and KEEPTTL defers to whatever the + // receiving store already holds — neither may gain a deadline. + let p = frame_of( + &Command::Set("k".into(), "v".into(), SetOptions::default()), + NOW, + ); + assert_eq!(p, vec!["SET", "k", "v"], "{p:?}"); + + let p = frame_of( + &Command::Set( + "k".into(), + "v".into(), + SetOptions { + expiry: Some(SetExpiry::KeepTtl), + ..Default::default() + }, + ), + NOW, + ); + assert_eq!(p, vec!["SET", "k", "v", "KEEPTTL"], "{p:?}"); + } + + #[test] + fn the_propagated_frame_is_a_command_the_replay_path_can_parse() { + // The frame is fed straight back through `Command::from_value` on AOF + // replay and on replicas, so an encoding no parser accepts would be a + // silent data-loss bug rather than a compile error. + for cmd in [ + Command::Set( + "k".into(), + "v".into(), + SetOptions { + expiry: Some(SetExpiry::Ex(5)), + ..Default::default() + }, + ), + Command::SetEx("k".into(), 5, "v".into()), + Command::PSetEx("k".into(), 5_000, "v".into()), + ] { + let f = broadcast_for(&cmd, &Value::SimpleString("OK".into()), NOW).unwrap(); + let (v, _) = Value::parse(&f).unwrap(); + let arr = match v { + Value::Push(i) | Value::Array(Some(i)) => Value::Array(Some(i)), + other => panic!("unexpected {other:?}"), + }; + let parsed = Command::from_value(arr).expect("replay must parse the frame"); + assert!( + matches!(&parsed, Command::Set(_, _, o) if matches!(o.expiry, Some(SetExpiry::Pxat(_)))), + "replayed command lost its absolute deadline: {parsed:?}" + ); + } + + let f = broadcast_for(&Command::Expire("k".into(), 5), &Value::Integer(1), NOW).unwrap(); + let (v, _) = Value::parse(&f).unwrap(); + let arr = match v { + Value::Push(i) => Value::Array(Some(i)), + other => panic!("unexpected {other:?}"), + }; + assert!(matches!( + Command::from_value(arr).unwrap(), + Command::PExpireAt(_, _) + )); + } + + /// The property the whole change exists for: the deadline is a point in + /// time, so *when* the frame is applied cannot change *when* it expires. + #[test] + fn replaying_the_same_write_later_does_not_extend_the_key() { + let cmd = Command::SetEx("k".into(), 60, "v".into()); + let frame = broadcast_for(&cmd, &Value::SimpleString("OK".into()), NOW).unwrap(); + + // Same write, propagated a full hour later, is a *different* deadline — + // but any single frame carries exactly one, whenever it is applied. + let later = + broadcast_for(&cmd, &Value::SimpleString("OK".into()), NOW + 3_600_000).unwrap(); + assert_ne!(frame, later); + assert_eq!(parts(&frame)[4], (NOW + 60_000).to_string()); + assert_eq!(parts(&later)[4], (NOW + 3_600_000 + 60_000).to_string()); + } + + /// The bug, end to end through the real AOF path: a key whose deadline has + /// already passed must stay dead when the log is replayed. + #[tokio::test] + async fn an_expired_key_is_not_resurrected_by_aof_replay() { + let path = tmp_path("expiry_replay.aof"); + let _ = tokio::fs::remove_file(&path).await; + let aof = AofWriter::open(path.clone(), AofSync::No).await.unwrap(); + + // A write whose 5-second TTL elapsed long ago — the shape of any + // short-lived key written before a restart that outlasted it. + let long_ago = now_unix_ms() - 3_600_000; + let frame = broadcast_for( + &Command::SetEx("session:revoked".into(), 5, "tok".into()), + &Value::SimpleString("OK".into()), + long_ago, + ) + .unwrap(); + aof.append(&frame).await; + // A live key, to prove replay still works at all. + let live = broadcast_for( + &Command::SetEx("session:live".into(), 600, "tok".into()), + &Value::SimpleString("OK".into()), + now_unix_ms(), + ) + .unwrap(); + aof.append(&live).await; + aof.flush().await; + + let store = KeyValueStore::new(); + assert_eq!(replay_aof(&store, &path).await, 2); + + assert_eq!( + store.execute(Command::Get("session:revoked".into())), + Value::BulkString(None), + "a key dead for an hour was resurrected by replay" + ); + assert_eq!( + store.execute(Command::Exists(vec!["session:revoked".into()])), + Value::Integer(0) + ); + assert_eq!( + store.execute(Command::Ttl("session:revoked".into())), + Value::Integer(-2) + ); + // ...while the key that had not expired survives with its remaining TTL. + assert_eq!( + store.execute(Command::Get("session:live".into())), + Value::BulkString(Some(b"tok".to_vec())) + ); + assert!( + matches!( + store.execute(Command::Ttl("session:live".into())), + Value::Integer(n) if (0..=600).contains(&n) + ), + "a live key must keep its original deadline, not gain a fresh one" + ); + + let _ = tokio::fs::remove_file(&path).await; + } + + /// EXPIRE has the same shape as SET..EX and the same failure mode. + #[tokio::test] + async fn an_elapsed_expire_does_not_extend_the_key_on_replay() { + let path = tmp_path("expiry_replay_expire.aof"); + let _ = tokio::fs::remove_file(&path).await; + let aof = AofWriter::open(path.clone(), AofSync::No).await.unwrap(); + + aof.append(b">3\r\n$3\r\nSET\r\n$1\r\nk\r\n$1\r\nv\r\n") + .await; + let long_ago = now_unix_ms() - 3_600_000; + let frame = broadcast_for( + &Command::Expire("k".into(), 30), + &Value::Integer(1), + long_ago, + ) + .unwrap(); + aof.append(&frame).await; + aof.flush().await; + + let store = KeyValueStore::new(); + replay_aof(&store, &path).await; + assert_eq!( + store.execute(Command::Get("k".into())), + Value::BulkString(None), + "an EXPIRE that elapsed before the restart was re-armed by replay" + ); + + let _ = tokio::fs::remove_file(&path).await; + } +} + +// ── Transaction abort ───────────────────────────────────────────────────────── + +/// Counters propagate by value, but must not clear the key's deadline. +/// +/// `INCR` is replicated as `SET key ` so a replica that missed a +/// frame converges on the primary's number rather than compounding its own — +/// but a bare `SET` also clears the TTL, and Redis's `INCR` leaves it alone. +/// The single most common expiring-counter idiom, `INCR key` + `EXPIRE key +/// window`, therefore replayed as a key with *no* expiry: the rate-limit +/// bucket, the per-minute quota and the retry counter all became permanent on +/// the replica, in the AOF and in every synced browser, and the window never +/// reset because the key it keyed on never went away. +#[cfg(test)] +mod counter_ttl_propagation_tests { + use super::*; + use core_engine::cmd::{SetExpiry, SetOptions}; + use core_engine::store::KeyValueStore; + + fn tmp_path(name: &str) -> PathBuf { + std::env::temp_dir().join(format!("recached_test_{name}_{}", std::process::id())) + } + + fn frame_args(cmd: &Command, response: &Value) -> Vec { + let f = broadcast_for(cmd, response, 0).expect("counter must propagate"); + let (v, _) = Value::parse(&f).unwrap(); + let items = match v { + Value::Push(i) | Value::Array(Some(i)) => i, + other => panic!("unexpected {other:?}"), + }; + items + .into_iter() + .map(|i| match i { + Value::BulkString(Some(b)) => String::from_utf8_lossy(&b).into_owned(), + other => panic!("unexpected {other:?}"), + }) + .collect() + } + + #[test] + fn every_counter_propagates_with_keepttl() { + for cmd in [ + Command::Incr("c".into()), + Command::Decr("c".into()), + Command::IncrBy("c".into(), 5), + Command::DecrBy("c".into(), 5), + ] { + let args = frame_args(&cmd, &Value::Integer(7)); + assert_eq!( + args, + vec!["SET", "c", "7", "KEEPTTL"], + "{} must not clear the key's deadline", + command_name(&cmd) + ); + } + } + + #[test] + fn a_counter_that_did_not_run_still_propagates_nothing() { + // INCR on a non-numeric value errors and changes nothing; replaying a + // SET for it would invent a value the primary never stored. + assert!( + broadcast_for( + &Command::Incr("c".into()), + &Value::Error("ERR not an integer".into()), + 0 + ) + .is_none() + ); + } + + #[test] + fn getset_still_clears_the_ttl() { + // GETSET *does* clear the TTL in Redis, so it must keep propagating a + // bare SET — the KEEPTTL change applies to counters only. + let args = frame_args( + &Command::GetSet("k".into(), "v".into()), + &Value::BulkString(None), + ); + assert_eq!(args, vec!["SET", "k", "v"]); + } + + /// The behaviour, through the real replay path: `INCR` + `EXPIRE` survives + /// a restart still holding its deadline. + #[tokio::test] + async fn an_expiring_counter_keeps_its_deadline_across_aof_replay() { + let path = tmp_path("counter_ttl.aof"); + let _ = tokio::fs::remove_file(&path).await; + let aof = AofWriter::open(path.clone(), AofSync::No).await.unwrap(); + + // The rate-limiter idiom: create with a window, then count into it. + let now = now_unix_ms(); + for frame in [ + broadcast_for( + &Command::Set( + "rate:user:42".into(), + "1".into(), + SetOptions { + expiry: Some(SetExpiry::Ex(60)), + ..Default::default() + }, + ), + &Value::SimpleString("OK".into()), + now, + ), + broadcast_for( + &Command::Incr("rate:user:42".into()), + &Value::Integer(2), + now, + ), + broadcast_for( + &Command::Incr("rate:user:42".into()), + &Value::Integer(3), + now, + ), + ] { + aof.append(&frame.unwrap()).await; + } + aof.flush().await; + + let store = KeyValueStore::new(); + replay_aof(&store, &path).await; + + assert_eq!( + store.execute(Command::Get("rate:user:42".into())), + Value::BulkString(Some(b"3".to_vec())), + "the counter must converge on the primary's value" + ); + match store.execute(Command::Ttl("rate:user:42".into())) { + Value::Integer(n) => assert!( + (1..=60).contains(&n), + "the rate-limit window was lost on replay — TTL is {n}, so the \ + bucket would never reset" + ), + other => panic!("expected an integer, got {other:?}"), + } + + let _ = tokio::fs::remove_file(&path).await; + } +} + +// ── Port configuration ──────────────────────────────────────────────────────── diff --git a/server-native/src/pubsub.rs b/server-native/src/pubsub.rs new file mode 100644 index 0000000..abca7d2 --- /dev/null +++ b/server-native/src/pubsub.rs @@ -0,0 +1,192 @@ +//! Pub/sub: the subscriber hub for channels and patterns, and the wire +//! encoding of the messages and acknowledgements it produces. + +use crate::*; + +pub(crate) enum PubSubMsg { + Message { + channel: String, + message: Vec, + }, + PMessage { + pattern: String, + channel: String, + message: Vec, + }, +} + +pub(crate) type PubSubSender = mpsc::UnboundedSender; + +pub(crate) struct PubSubHub { + pub(crate) channel_subs: HashMap>, + pub(crate) pattern_subs: Vec<(String, u64, PubSubSender)>, +} + +impl PubSubHub { + pub(crate) fn new() -> Self { + Self { + channel_subs: HashMap::new(), + pattern_subs: Vec::new(), + } + } + + pub(crate) fn subscribe(&mut self, conn_id: u64, channel: &str, tx: PubSubSender) { + self.channel_subs + .entry(channel.to_string()) + .or_default() + .push((conn_id, tx)); + } + + pub(crate) fn psubscribe(&mut self, conn_id: u64, pattern: &str, tx: PubSubSender) { + self.pattern_subs.push((pattern.to_string(), conn_id, tx)); + } + + pub(crate) fn unsubscribe(&mut self, conn_id: u64, channel: &str) { + if let Some(v) = self.channel_subs.get_mut(channel) { + v.retain(|(id, _)| *id != conn_id); + if v.is_empty() { + self.channel_subs.remove(channel); + } + } + } + + pub(crate) fn punsubscribe(&mut self, conn_id: u64, pattern: &str) { + self.pattern_subs + .retain(|(p, id, _)| !(p == pattern && *id == conn_id)); + } + + pub(crate) fn unsubscribe_all(&mut self, conn_id: u64) { + self.channel_subs.retain(|_, v| { + v.retain(|(id, _)| *id != conn_id); + !v.is_empty() + }); + self.pattern_subs.retain(|(_, id, _)| *id != conn_id); + } + + /// Channels with at least one live subscriber, for `PUBSUB CHANNELS`. + /// + /// `unsubscribe` and `unsubscribe_all` remove a channel's entry once its + /// last subscriber leaves, and `publish` drops senders whose receiver has + /// closed, so a key in `channel_subs` implies a live subscriber. The + /// `is_empty` guard covers the one window where it does not: a connection + /// that died between the last publish and its close handler. + pub(crate) fn active_channels(&self) -> impl Iterator { + self.channel_subs + .iter() + .filter(|(_, subs)| !subs.is_empty()) + .map(|(channel, _)| channel) + } + + /// Subscribers to one exact channel, for `PUBSUB NUMSUB`. Pattern + /// subscribers are deliberately not counted, matching Redis: a `PSUBSCRIBE` + /// is reported by `NUMPAT`, and counting it here would double-count a + /// client that holds both. + pub(crate) fn subscriber_count(&self, channel: &str) -> i64 { + self.channel_subs + .get(channel) + .map(|subs| subs.len() as i64) + .unwrap_or(0) + } + + /// Distinct patterns under subscription, for `PUBSUB NUMPAT`. Distinct is + /// the Redis definition: two clients on `news.*` are one pattern, not two. + pub(crate) fn pattern_count(&self) -> i64 { + let mut seen: Vec<&str> = self + .pattern_subs + .iter() + .map(|(p, _, _)| p.as_str()) + .collect(); + seen.sort_unstable(); + seen.dedup(); + seen.len() as i64 + } + + /// Deliver to all matching subscribers; returns the count delivered. + pub(crate) fn publish(&mut self, channel: &str, message: &[u8]) -> i64 { + let mut count = 0i64; + + if let std::collections::hash_map::Entry::Occupied(mut e) = + self.channel_subs.entry(channel.to_string()) + { + let subs = e.get_mut(); + subs.retain(|(_, tx)| { + let ok = tx + .send(PubSubMsg::Message { + channel: channel.to_string(), + message: message.to_vec(), + }) + .is_ok(); + if ok { + count += 1; + } + ok + }); + if subs.is_empty() { + e.remove(); + } + } + + let pattern_txs: Vec<(String, PubSubSender)> = self + .pattern_subs + .iter() + .filter(|(p, _, _)| core_engine::store::glob_match(p, channel)) + .map(|(p, _, tx)| (p.clone(), tx.clone())) + .collect(); + for (pattern, tx) in pattern_txs { + if tx + .send(PubSubMsg::PMessage { + pattern, + channel: channel.to_string(), + message: message.to_vec(), + }) + .is_ok() + { + count += 1; + } + } + self.pattern_subs.retain(|(_, _, tx)| !tx.is_closed()); + count + } +} + +pub(crate) type SharedPubSub = Arc>; + +// ── observable keys ─────────────────────────────────────────────────────────── + +pub(crate) fn encode_pubsub_msg(msg: PubSubMsg, protover: u8) -> Vec { + let frame = |parts: Vec| { + if protover >= 3 { + Value::Push(parts) + } else { + Value::Array(Some(parts)) + } + }; + match msg { + PubSubMsg::Message { channel, message } => frame(vec![ + Value::BulkString(Some(b"message".to_vec())), + Value::BulkString(Some(channel.into_bytes())), + Value::BulkString(Some(message)), + ]) + .serialize(), + PubSubMsg::PMessage { + pattern, + channel, + message, + } => frame(vec![ + Value::BulkString(Some(b"pmessage".to_vec())), + Value::BulkString(Some(pattern.into_bytes())), + Value::BulkString(Some(channel.into_bytes())), + Value::BulkString(Some(message)), + ]) + .serialize(), + } +} + +pub(crate) fn resp_subscribe_ack(kind: &str, channel: &str, count: usize) -> Vec { + Value::Array(Some(vec![ + Value::BulkString(Some(kind.as_bytes().to_vec())), + Value::BulkString(Some(channel.as_bytes().to_vec())), + Value::Integer(count as i64), + ])) + .serialize() +} diff --git a/server-native/src/replication.rs b/server-native/src/replication.rs new file mode 100644 index 0000000..9327d26 --- /dev/null +++ b/server-native/src/replication.rs @@ -0,0 +1,661 @@ +//! Replication: the listener that serves replicas, the client that follows a +//! primary, and the authentication throttle guarding the handshake. + +use crate::*; + +pub(crate) type ReplSender = mpsc::Sender>; + +/// A connected replica: its write channel plus the counters that make lag +/// observable. +/// +/// Replication was previously one-way, so the primary could only report how +/// many replicas were attached — never how far behind one had fallen. The +/// replica now acknowledges each applied frame, and the difference between +/// what was queued and what was acknowledged is the lag. +pub(crate) struct ReplicaHandle { + pub(crate) tx: ReplSender, + /// Frames handed to this replica's channel. + pub(crate) sent: Arc, + /// Frames the replica reports as applied. + pub(crate) acked: Arc, +} + +/// Connected-replica registry. `count` mirrors `senders.len()` (updated by +/// every writer while holding the lock) so the per-write hot path can skip +/// the mutex entirely when no replica is connected. +pub(crate) struct ReplHub { + pub(crate) senders: tokio::sync::Mutex>, + pub(crate) count: AtomicUsize, +} + +impl ReplHub { + /// Deepest send queue across connected replicas, in frames. + /// + /// Replication is fire-and-forget — replicas never acknowledge an applied + /// offset — so true offset lag is not observable without a protocol change. + /// Queue depth is the honest proxy available today: a replica that cannot + /// keep up backs its channel up, and a queue at capacity means frames are + /// about to be dropped. + pub(crate) async fn max_queue_depth(&self) -> usize { + let senders = self.senders.lock().await; + senders + .iter() + .map(|r| r.tx.max_capacity().saturating_sub(r.tx.capacity())) + .max() + .unwrap_or(0) + } + + /// Send one frame to every attached replica, dropping any that cannot keep + /// up. Increments each surviving replica's sent counter, which is one half + /// of the lag calculation. + pub(crate) async fn fan_out(&self, bytes: Vec) { + let mut reg = self.senders.lock().await; + reg.retain(|r| match r.tx.try_send(bytes.clone()) { + Ok(()) => { + r.sent.fetch_add(1, Ordering::Relaxed); + true + } + Err(mpsc::error::TrySendError::Full(_)) => { + warn!( + "Replica fell too far behind (channel full) — disconnecting so it can resync" + ); + false + } + Err(mpsc::error::TrySendError::Closed(_)) => false, + }); + self.count.store(reg.len(), Ordering::Relaxed); + } + + /// Frames the furthest-behind replica has yet to acknowledge. + /// + /// This is true lag: how much of what the primary sent has actually been + /// applied downstream. Queue depth only shows what is stuck locally, and + /// reads zero for a replica that has received frames but cannot apply them. + pub(crate) async fn max_lag_frames(&self) -> u64 { + let senders = self.senders.lock().await; + senders + .iter() + .map(|r| { + r.sent + .load(Ordering::Relaxed) + .saturating_sub(r.acked.load(Ordering::Relaxed)) + }) + .max() + .unwrap_or(0) + } + + pub(crate) fn new() -> ReplRegistry { + Arc::new(ReplHub { + senders: tokio::sync::Mutex::new(Vec::new()), + count: AtomicUsize::new(0), + }) + } + + pub(crate) fn is_empty(&self) -> bool { + self.count.load(Ordering::Relaxed) == 0 + } +} + +pub(crate) type ReplRegistry = Arc; + +/// Default per-replica channel capacity (number of pending write frames). +/// When a replica falls this many writes behind the primary it is disconnected +/// so it can reconnect and receive a fresh snapshot — the primary write path +/// is never blocked. +pub(crate) const DEFAULT_REPL_CHANNEL_CAPACITY: usize = 4096; + +/// Upper bound on a single length-prefixed replication frame (snapshot or +/// command). The replication port may be unauthenticated and plaintext, so an +/// untrusted peer could otherwise send a 4 GB length prefix and force a matching +/// allocation. 512 MB comfortably covers a large snapshot while bounding abuse. +pub(crate) const MAX_REPL_FRAME_BYTES: usize = 512 * 1024 * 1024; + +// ── Server state ────────────────────────────────────────────────────────────── + +/// Per-peer replication auth throttle. +/// +/// The RESP port drops a connection after `MAX_AUTH_FAILURES` guesses, but the +/// replication handshake is one-shot: a wrong password costs the attacker a +/// single TCP connection and nothing else, so the port offered effectively +/// unlimited guesses at a secret that yields the entire keyspace. Failures are +/// counted per source address over a rolling window, and a peer that exhausts +/// them is refused before the handshake is read at all. +/// +/// Keyed by address rather than by connection, which is the whole point — the +/// weakness being closed is that reconnecting reset the count. +pub(crate) struct ReplAuthThrottle { + pub(crate) failures: std::sync::Mutex>, +} + +impl ReplAuthThrottle { + pub(crate) fn new() -> Arc { + Arc::new(Self { + failures: std::sync::Mutex::new(HashMap::new()), + }) + } + + /// True when this peer has spent its attempts and must be refused. + pub(crate) fn is_blocked(&self, ip: IpAddr) -> bool { + let Ok(map) = self.failures.lock() else { + return false; + }; + match map.get(&ip) { + Some((count, last)) => *count >= MAX_AUTH_FAILURES && last.elapsed() < REPL_AUTH_WINDOW, + None => false, + } + } + + pub(crate) fn record_failure(&self, ip: IpAddr) { + let Ok(mut map) = self.failures.lock() else { + return; + }; + let now = std::time::Instant::now(); + // Sweep before inserting so a spray across many source addresses cannot + // grow the map without bound. + if map.len() >= REPL_AUTH_SWEEP_THRESHOLD { + map.retain(|_, (_, last)| last.elapsed() < REPL_AUTH_WINDOW); + } + let entry = map.entry(ip).or_insert((0, now)); + // A peer that went quiet for longer than the window starts over, so a + // slow trickle is not punished forever. + if entry.1.elapsed() >= REPL_AUTH_WINDOW { + *entry = (0, now); + } + entry.0 = entry.0.saturating_add(1); + entry.1 = now; + } + + pub(crate) fn record_success(&self, ip: IpAddr) { + if let Ok(mut map) = self.failures.lock() { + map.remove(&ip); + } + } +} + +/// Read the newline-terminated replication auth line. +/// +/// Reads until the terminator rather than reading exactly `password.len() + 1` +/// bytes, which is how the previous implementation worked: the number of bytes +/// the server waited for *was* the password length, so an attacker could +/// recover it exactly by drip-feeding one byte at a time and watching when the +/// server replied. +pub(crate) async fn read_repl_auth_line(socket: &mut S) -> std::io::Result> +where + S: AsyncRead + Unpin, +{ + let mut line = Vec::with_capacity(64); + let mut byte = [0u8; 1]; + loop { + socket.read_exact(&mut byte).await?; + if byte[0] == b'\n' { + return Ok(line); + } + if line.len() >= MAX_REPL_AUTH_LINE { + return Err(std::io::Error::new( + ErrorKind::InvalidData, + "replication auth line too long", + )); + } + line.push(byte[0]); + } +} + +#[allow(clippy::too_many_arguments)] +pub(crate) async fn run_repl_server( + bind_host: String, + port: u16, + store: Arc, + snap_cfg: Arc, + replicas: ReplRegistry, + repl_password: Option>, + repl_channel_capacity: usize, + allowed_ips: Option>>, + semaphore: Arc, + throttle: Arc, + tls: Arc>, +) { + let listener = match TcpListener::bind(format!("{}:{}", bind_host, port)).await { + Ok(l) => l, + Err(e) => { + warn!("Replication listener failed to bind :{}: {}", port, e); + return; + } + }; + info!( + "Replication server listening on {}:{} ({})", + bind_host, + port, + if tls.is_some() { "TLS" } else { "plaintext" } + ); + loop { + match listener.accept().await { + Ok((socket, addr)) => { + // The IP allowlist and the connection limit were previously + // applied on the RESP and WebSocket listeners only, so neither + // constrained the one port that streams the whole keyspace. + if let Some(allowed) = &allowed_ips + && !allowed.contains(&addr.ip()) + { + debug!("Replication: rejected IP {}", addr.ip()); + continue; + } + if throttle.is_blocked(addr.ip()) { + warn!( + "Replication: {} refused — too many failed auth attempts", + addr.ip() + ); + continue; + } + let permit = match Arc::clone(&semaphore).try_acquire_owned() { + Ok(p) => p, + Err(_) => { + warn!("Replication: connection limit reached, dropping {}", addr); + continue; + } + }; + info!("Replica connected from {}", addr); + let store = Arc::clone(&store); + let snap_cfg = Arc::clone(&snap_cfg); + let replicas = Arc::clone(&replicas); + let pwd = repl_password.clone(); + let thr = Arc::clone(&throttle); + let tls = Arc::clone(&tls); + tokio::spawn(async move { + let _permit = permit; + // Bounded like the other listeners: the permit is already + // held, so a peer that never negotiates must not keep it. + let outcome = if let Some(acceptor) = tls.as_ref() { + match tokio::time::timeout(handshake_timeout(), acceptor.accept(socket)) + .await + { + Ok(Ok(stream)) => { + handle_replica( + stream, + store, + snap_cfg, + replicas, + pwd, + repl_channel_capacity, + addr.ip(), + thr, + ) + .await + } + Ok(Err(e)) => { + // The most likely cause by far is a replica that + // has not been given RECACHED_REPL_TLS_CA, which + // otherwise looks like an unexplained disconnect. + warn!( + "Replication TLS handshake failed from {}: {} — is that \ + replica configured with RECACHED_REPL_TLS_CA?", + addr, e + ); + return; + } + Err(_) => { + debug!("Replication TLS handshake from {} timed out", addr); + return; + } + } + } else { + handle_replica( + socket, + store, + snap_cfg, + replicas, + pwd, + repl_channel_capacity, + addr.ip(), + thr, + ) + .await + }; + if let Err(e) = outcome { + info!("Replica {} disconnected: {}", addr, e); + } + }); + } + Err(e) => warn!("Replication accept error: {}", e), + } + } +} + +#[allow(clippy::too_many_arguments)] +pub(crate) async fn handle_replica( + mut socket: S, + store: Arc, + _snap_cfg: Arc, + replicas: ReplRegistry, + repl_password: Option>, + repl_channel_capacity: usize, + peer_ip: IpAddr, + throttle: Arc, +) -> std::io::Result<()> +where + S: AsyncRead + AsyncWrite + Unpin + Send, +{ + // 0. Auth handshake — replica must send "\n" before anything else. + // + // Bounded by a deadline as well as a length: a peer that connects and then + // says nothing would otherwise hold its connection permit forever. + if let Some(pwd) = &repl_password { + let line = match tokio::time::timeout(handshake_timeout(), read_repl_auth_line(&mut socket)) + .await + { + Ok(res) => res?, + Err(_) => { + return Err(std::io::Error::new( + ErrorKind::TimedOut, + "replication auth handshake timed out", + )); + } + }; + if !ct_eq_bytes(&line, pwd.as_bytes()) { + throttle.record_failure(peer_ip); + let _ = socket + .write_all(b"-ERR invalid replication password\n") + .await; + return Err(std::io::Error::new( + ErrorKind::PermissionDenied, + "replication auth failed", + )); + } + throttle.record_success(peer_ip); + socket.write_all(b"+OK\n").await?; + socket.flush().await?; + } + + // 1. Register channel first so subsequent writes are buffered + let (tx, mut rx) = mpsc::channel::>(repl_channel_capacity); + let sent = Arc::new(AtomicU64::new(0)); + let acked = Arc::new(AtomicU64::new(0)); + { + let mut reg = replicas.senders.lock().await; + reg.push(ReplicaHandle { + tx, + sent: Arc::clone(&sent), + acked: Arc::clone(&acked), + }); + replicas.count.store(reg.len(), Ordering::Relaxed); + } + + // 2. Take snapshot and send (writes since snapshot are in channel) + let snap_bytes = + rmp_serde::to_vec(&store.snapshot()).map_err(|e| std::io::Error::other(e.to_string()))?; + let len = snap_bytes.len() as u32; + socket.write_all(&len.to_le_bytes()).await?; + socket.write_all(&snap_bytes).await?; + socket.flush().await?; + + // 3. Stream buffered + ongoing writes, and read acknowledgements + // + // The socket is bidirectional but used to carry frames one way only, which + // left the primary unable to say how far behind a replica was. The replica + // now writes back a cumulative count of applied frames; `sent - acked` is + // the lag. Reading and writing are selected over so a replica that stops + // acknowledging cannot stall the write side, and vice versa. + let (mut rd, mut wr) = tokio::io::split(socket); + let mut ack_buf = [0u8; 8]; + loop { + tokio::select! { + frame = rx.recv() => { + let Some(bytes) = frame else { break }; + let len = bytes.len() as u32; + wr.write_all(&len.to_le_bytes()).await?; + wr.write_all(&bytes).await?; + wr.flush().await?; + } + res = rd.read_exact(&mut ack_buf) => { + // A replica that closes its read side, or one running a build + // that predates acks, simply stops updating the gauge — it is + // not an error, so the stream continues either way. + if res.is_err() { + break; + } + let applied = u64::from_le_bytes(ack_buf); + // Monotonic: a reordered or replayed ack must never walk the + // high-water mark backwards and report negative lag. + acked.fetch_max(applied, Ordering::Relaxed); + } + } + } + Ok(()) +} + +// ── Replication client (replica side) ──────────────────────────────────────── + +pub(crate) async fn run_repl_client( + primary_addr: String, + store: Arc, + state: Arc, + repl_password: Option, + failover_timeout_secs: Option, + tx: broadcast::Sender, + tls: Option<(TlsConnector, String)>, +) { + let mut backoff_secs = 2u64; + let mut unreachable_since: Option = None; + + loop { + // Stop if already promoted (manual REPLICAOF NO ONE or earlier auto-promotion). + if !state.is_replica() { + return; + } + + info!("Replica: connecting to primary at {}", primary_addr); + match TcpStream::connect(&primary_addr).await { + Err(e) => { + warn!("Replica: connect failed: {}", e); + unreachable_since.get_or_insert_with(std::time::Instant::now); + } + Ok(socket) => { + // Primary is reachable — reset the unreachable timer. + unreachable_since = None; + backoff_secs = 2; + + // TLS is what makes the primary's *identity* checked, not just + // the channel encrypted: without it a DNS hijack or an on-path + // attacker can feed this replica an arbitrary keyspace, and the + // replica has no way to tell. + let result = match &tls { + None => { + sync_from_primary( + &mut { socket }, + &store, + repl_password.as_deref(), + &tx, + &state, + ) + .await + } + Some((connector, servername)) => { + match ServerName::try_from(servername.clone()) { + Err(_) => { + error!( + "Replica: '{}' is not a valid TLS server name — set \ + RECACHED_REPL_TLS_SERVERNAME to the name on the primary's \ + certificate", + servername + ); + return; + } + Ok(name) => { + match tokio::time::timeout( + handshake_timeout(), + connector.connect(name, socket), + ) + .await + { + Err(_) => Err(std::io::Error::new( + ErrorKind::TimedOut, + "TLS handshake with primary timed out", + )), + Ok(Err(e)) => Err(std::io::Error::other(format!( + "TLS handshake with primary failed: {e} — check that the \ + primary has RECACHED_TLS_CERT set and that \ + RECACHED_REPL_TLS_CA trusts it" + ))), + Ok(Ok(mut stream)) => { + sync_from_primary( + &mut stream, + &store, + repl_password.as_deref(), + &tx, + &state, + ) + .await + } + } + } + } + } + }; + + if let Err(e) = result { + warn!("Replica: sync ended: {}", e); + // Sync dropped — primary may be gone; start tracking if not already. + unreachable_since.get_or_insert_with(std::time::Instant::now); + } + } + } + + // Auto-failover: promote if primary has been unreachable long enough. + if let (Some(timeout), Some(since)) = (failover_timeout_secs, unreachable_since) { + let elapsed = since.elapsed().as_secs(); + if elapsed >= timeout { + warn!( + "Replica: primary unreachable for {}s (timeout {}s) — auto-promoting to primary", + elapsed, timeout + ); + state.promote_to_primary(); + return; + } + info!( + "Replica: primary unreachable for {}s / {}s before auto-failover", + elapsed, timeout + ); + } + + tokio::time::sleep(tokio::time::Duration::from_secs(backoff_secs)).await; + backoff_secs = (backoff_secs * 2).min(30); + } +} + +pub(crate) async fn sync_from_primary( + socket: &mut S, + store: &KeyValueStore, + repl_password: Option<&str>, + tx: &broadcast::Sender, + state: &ServerState, +) -> std::io::Result<()> +where + S: AsyncRead + AsyncWrite + Unpin, +{ + // 0. Send auth password if configured + if let Some(pwd) = repl_password { + let msg = format!("{}\n", pwd); + socket.write_all(msg.as_bytes()).await?; + socket.flush().await?; + // Read "+OK\n" (4 bytes) + let mut resp = [0u8; 4]; + socket.read_exact(&mut resp).await?; + if &resp != b"+OK\n" { + return Err(std::io::Error::new( + ErrorKind::PermissionDenied, + "replication auth rejected by primary", + )); + } + } + + // 1. Receive full snapshot + let mut len_buf = [0u8; 4]; + socket.read_exact(&mut len_buf).await?; + let snap_len = u32::from_le_bytes(len_buf) as usize; + if snap_len > MAX_REPL_FRAME_BYTES { + return Err(std::io::Error::new( + ErrorKind::InvalidData, + format!("snapshot frame too large ({snap_len} > {MAX_REPL_FRAME_BYTES} bytes)"), + )); + } + let mut snap_bytes = vec![0u8; snap_len]; + socket.read_exact(&mut snap_bytes).await?; + + match rmp_serde::from_slice::>(&snap_bytes) { + Ok(entries) => { + let count = entries.len(); + store.restore(entries); + info!("Replica: snapshot loaded ({} entries)", count); + } + Err(e) => { + return Err(std::io::Error::new(ErrorKind::InvalidData, e.to_string())); + } + } + + // 2. Stream write commands from primary, acknowledging what we apply + // + // Every frame is counted, including one that fails to parse: the primary + // counts frames it sent, so skipping a bad frame here would desynchronise + // the two offsets and understate lag forever after. + let mut applied: u64 = 0; + loop { + let mut len_buf = [0u8; 4]; + socket.read_exact(&mut len_buf).await?; + let cmd_len = u32::from_le_bytes(len_buf) as usize; + if cmd_len > MAX_REPL_FRAME_BYTES { + return Err(std::io::Error::new( + ErrorKind::InvalidData, + format!("command frame too large ({cmd_len} > {MAX_REPL_FRAME_BYTES} bytes)"), + )); + } + let mut cmd_bytes = vec![0u8; cmd_len]; + socket.read_exact(&mut cmd_bytes).await?; + applied += 1; + + match Value::parse(&cmd_bytes) { + Ok((value, _)) => { + // Replication frames are broadcast as RESP3 Push (>N\r\n); normalise to + // Array so Command::from_value can parse them. + let normalised = match value { + Value::Push(inner) => Value::Array(Some(inner)), + other => other, + }; + if let Ok(cmd) = Command::from_value(normalised) { + let keys = primary_keys(&cmd); + store.execute(cmd); + // Relay the applied write so this replica's own WebSocket + // clients see it, and any sub-replicas / AOF get it too + // (enables multi-tier replication and replica WS push). + let _ = tx.send(Arc::new(SyncPush { + origin: 0, + keys, + resp: cmd_bytes.clone(), + })); + state.on_write(&cmd_bytes).await; + } + } + Err(e) => warn!("Replica: bad command from primary: {}", e), + } + + // Acknowledge on the same socket. TcpStream is unbuffered, so this is a + // single 8-byte write with no flush; a failure means the primary is + // gone, which the next read will surface with a better error. + if socket.write_all(&applied.to_le_bytes()).await.is_err() { + warn!("Replica: failed to send replication acknowledgement"); + } + } +} + +// ── security helpers ───────────────────────────────────────────────────────── + +/// Constant-time byte slice equality to prevent timing-based password leaks. +pub(crate) fn ct_eq_bytes(a: &[u8], b: &[u8]) -> bool { + if a.len() != b.len() { + return false; + } + a.iter() + .zip(b.iter()) + .fold(0u8, |acc, (x, y)| acc | (x ^ y)) + == 0 +} + +// ── sync scoping ───────────────────────────────────────────────────────────── diff --git a/server-native/src/server_state.rs b/server-native/src/server_state.rs new file mode 100644 index 0000000..3967b07 --- /dev/null +++ b/server-native/src/server_state.rs @@ -0,0 +1,185 @@ +//! The shared server state every connection writes through: AOF handle, +//! replica registry, replica/primary role, and write de-duplication. + +use crate::*; + +pub(crate) struct ServerState { + pub(crate) snap: Arc, + pub(crate) aof: Option>, + pub(crate) replicas: ReplRegistry, + /// true = currently acting as a read-only replica + pub(crate) is_replica: std::sync::atomic::AtomicBool, + /// Exactly-once bookkeeping for DEDUP-wrapped writes: client id → + /// (highest id applied, last-seen ms). Clients send monotonically + /// increasing ids and replay in order, so a single high-water mark per + /// client suffices — no seen-set. In-memory only: a server restart + /// reopens the (already narrow) duplicate window, which is documented. + pub(crate) dedup: std::sync::Mutex>, + /// Ephemeral (`ESET`) keys → the connection that currently owns them. + /// + /// Ownership transfers on each `ESET`, which is what makes multiple tabs + /// work: two tabs both setting `presence:user:42` leave the *later* one as + /// owner, so the first tab closing does not mark the user offline. Only the + /// owning connection's close deletes the key. + pub(crate) ephemeral: std::sync::Mutex>, + /// Set when a dedup high-water mark advances; cleared once persisted. + pub(crate) dedup_dirty: std::sync::atomic::AtomicBool, +} + +impl ServerState { + /// Record `conn_id` as the owner of an ephemeral key, replacing any + /// previous owner. + pub(crate) fn claim_ephemeral(&self, key: &str, conn_id: u64) { + if let Ok(mut map) = self.ephemeral.lock() { + map.insert(key.to_string(), conn_id); + } + } + + /// Keys still owned by `conn_id`, removed from the registry. Called once + /// when a connection closes. + pub(crate) fn take_ephemeral_for(&self, conn_id: u64) -> Vec { + let Ok(mut map) = self.ephemeral.lock() else { + return Vec::new(); + }; + let owned: Vec = map + .iter() + .filter(|(_, id)| **id == conn_id) + .map(|(k, _)| k.clone()) + .collect(); + for k in &owned { + map.remove(k); + } + owned + } +} + +/// Sweep dedup client entries idle longer than this once the map is large. +pub(crate) const DEDUP_IDLE_MS: u64 = 24 * 60 * 60 * 1000; + +pub(crate) const DEDUP_SWEEP_THRESHOLD: usize = 10_000; + +impl ServerState { + pub(crate) fn is_replica(&self) -> bool { + self.is_replica.load(Ordering::Relaxed) + } + + pub(crate) fn promote_to_primary(&self) { + self.is_replica.store(false, Ordering::Relaxed); + info!("REPLICAOF NO ONE: promoted to primary — writes now accepted"); + } + + /// True when a write must be RESP-encoded for the durability/replication + /// path even if no other consumer needs it. + pub(crate) fn needs_write_log(&self) -> bool { + self.aof.is_some() || !self.replicas.is_empty() + } + + /// Record a DEDUP-wrapped write. Returns `true` when `id` was already + /// applied for this client (the write must be skipped). Marks the id + /// *before* execution so a crash between check and execute can never + /// double-apply. + pub(crate) fn dedup_seen(&self, client: &str, id: u64) -> bool { + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() as u64; + let mut map = self.dedup.lock().expect("dedup mutex poisoned"); + if map.len() > DEDUP_SWEEP_THRESHOLD { + map.retain(|_, (_, seen)| now.saturating_sub(*seen) < DEDUP_IDLE_MS); + } + match map.get_mut(client) { + Some((hwm, seen)) => { + *seen = now; + if id <= *hwm { + true + } else { + *hwm = id; + self.dedup_dirty.store(true, Ordering::Relaxed); + false + } + } + None => { + map.insert(client.to_string(), (id, now)); + self.dedup_dirty.store(true, Ordering::Relaxed); + false + } + } + } + + /// Called after every successful write: appends to AOF and fans out to replicas. + pub(crate) async fn on_write(&self, resp: &[u8]) { + if let Some(aof) = &self.aof { + aof.append(resp).await; + } + if self.replicas.is_empty() { + return; + } + self.replicas.fan_out(resp.to_vec()).await; + } + + /// Path of the dedup sidecar, alongside the snapshot. + pub(crate) fn dedup_path(&self) -> std::path::PathBuf { + self.snap.path.with_extension("dedup") + } + + /// Persist dedup high-water marks so exactly-once delivery survives a + /// restart. Written atomically (temp + rename) and only when a mark has + /// advanced. The map is one `u64` per client, so this stays small enough to + /// flush far more often than the snapshot. + pub(crate) async fn persist_dedup(&self) { + if !self.dedup_dirty.swap(false, Ordering::Relaxed) { + return; + } + let marks: Vec<(String, u64)> = match self.dedup.lock() { + Ok(map) => map.iter().map(|(c, (hwm, _))| (c.clone(), *hwm)).collect(), + Err(_) => return, + }; + let path = self.dedup_path(); + let tmp = temp_sibling(&path, "dedup"); + match rmp_serde::to_vec(&marks) { + Err(e) => warn!("Dedup serialize failed: {}", e), + Ok(bytes) => match write_private(&tmp, &bytes).await { + Err(e) => warn!("Dedup write failed: {}", e), + Ok(()) => match tokio::fs::rename(&tmp, &path).await { + Err(e) => warn!("Dedup rename failed: {}", e), + Ok(()) => sync_parent_dir(&path).await, + }, + }, + } + } + + /// Restore dedup marks at boot. `seen` timestamps are not persisted — they + /// only drive idle sweeping, so restored entries start their idle clock now. + pub(crate) async fn load_dedup(&self) { + let path = self.dedup_path(); + let Ok(bytes) = tokio::fs::read(&path).await else { + return; + }; + match rmp_serde::from_slice::>(&bytes) { + Err(e) => warn!("Dedup sidecar unreadable ({}), ignoring: {:?}", e, path), + Ok(marks) => { + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() as u64; + if let Ok(mut map) = self.dedup.lock() { + let count = marks.len(); + for (client, hwm) in marks { + map.insert(client, (hwm, now)); + } + info!("Restored {} dedup high-water mark(s)", count); + } + } + } + } + + /// Save snapshot, reset the dirty counter, then truncate AOF (snapshot subsumes the log). + pub(crate) async fn save(&self, store: &KeyValueStore) { + self.persist_dedup().await; + save_snapshot(store, &self.snap).await; + store.reset_dirty(); + if let Some(aof) = &self.aof { + aof.truncate().await; + } + } +} diff --git a/server-native/src/sync_scopes.rs b/server-native/src/sync_scopes.rs new file mode 100644 index 0000000..1dbe2eb --- /dev/null +++ b/server-native/src/sync_scopes.rs @@ -0,0 +1,318 @@ +//! Sync scopes: signed tokens restricting a WebSocket connection to a set of +//! key patterns, and the per-command classification they are checked against. + +use crate::*; + +/// One mutation pushed towards WebSocket peers: the RESP push frame plus the +/// keys it touches, so each connection can filter against its sync scopes +/// without re-parsing the frame. Wrapped in `Arc` — the broadcast channel +/// clones the payload once per receiver, so a clone is a refcount bump. +pub(crate) struct SyncPush { + pub(crate) origin: u64, + pub(crate) keys: Vec, + pub(crate) resp: Vec, +} + +pub(crate) type SyncMsg = Arc; + +/// True when a mutation touching `keys` is visible to a connection whose sync +/// scopes are `scopes`. A mutation with no keys (FLUSHDB) affects every scope. +pub(crate) fn scopes_match(scopes: &[String], keys: &[String]) -> bool { + keys.is_empty() + || keys + .iter() + .any(|k| scopes.iter().any(|p| core_engine::store::glob_match(p, k))) +} + +/// Verify a signed sync-scope token and return the granted patterns. +/// +/// Token format: `base64url(payload) "." base64url(hmac_sha256(secret, base64url(payload)))` +/// where payload is comma-separated glob patterns with an optional +/// `|` suffix. The HMAC is computed over the *encoded* +/// payload string, so minting in JS is one `createHmac` call on the base64url +/// text — no byte-level canonicalisation questions. +pub(crate) fn verify_sync_token(secret: &str, token: &str) -> Result, &'static str> { + use base64::Engine as _; + use hmac::{Hmac, Mac}; + use sha2::Sha256; + + let engine = base64::engine::general_purpose::URL_SAFE_NO_PAD; + let (payload_b64, sig_b64) = token.split_once('.').ok_or("malformed token")?; + let sig = engine.decode(sig_b64).map_err(|_| "malformed signature")?; + let mut mac = + Hmac::::new_from_slice(secret.as_bytes()).expect("HMAC accepts any key length"); + mac.update(payload_b64.as_bytes()); + if !ct_eq_bytes(&sig, &mac.finalize().into_bytes()) { + return Err("invalid signature"); + } + let payload_bytes = engine + .decode(payload_b64) + .map_err(|_| "malformed payload")?; + let payload = String::from_utf8(payload_bytes).map_err(|_| "malformed payload")?; + let (patterns_str, expiry) = match payload.split_once('|') { + Some((p, e)) => (p, Some(e)), + None => (payload.as_str(), None), + }; + if let Some(e) = expiry { + let exp: u64 = e.parse().map_err(|_| "malformed expiry")?; + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs(); + if now >= exp { + return Err("token expired"); + } + } + let patterns: Vec = patterns_str + .split(',') + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(String::from) + .collect(); + if patterns.is_empty() { + return Err("token grants no patterns"); + } + // Token patterns reach `glob_match` without passing through the command + // parser, so the cap `check_pattern` applies to `KEYS`/`SCAN`/`PSUBSCRIBE` + // has to be repeated here. These are matched once per key per write, so an + // over-long one is the most expensive place to put a pattern — and a + // compromised or careless minting service should not be able to. + if patterns + .iter() + .any(|p| p.len() > core_engine::store::MAX_PATTERN_BYTES) + { + return Err("token grants an over-long pattern"); + } + Ok(patterns) +} + +/// What a command touches, for scope enforcement on token-scoped WebSocket +/// connections. +#[derive(Debug)] +pub(crate) enum CommandScope { + /// No key access (PING, AUTH, MULTI, SYNC, pub/sub) — always allowed. + KeyLess, + /// Touches exactly these keys — every one must match a scope pattern. + Keys(Vec), + /// Keyspace-wide or administrative — denied on scoped connections. + Admin, +} + +pub(crate) fn command_scope(cmd: &Command) -> CommandScope { + match cmd { + Command::Ping(_) + | Command::Auth(_) + | Command::Hello(_) + | Command::Multi + | Command::Exec + | Command::Discard + | Command::Subscribe(_) + | Command::Unsubscribe(_) + | Command::PSubscribe(_) + | Command::PUnsubscribe(_) + | Command::Publish(_, _) + | Command::Sync(_) + // QSUB patterns are scope-checked against the grant in the WS handler. + | Command::QSub(_) + | Command::QUnsub(_) + // QUIT and CLIENT describe the connection itself, which a scoped + // connection is entitled to know about; COMMAND describes the server's + // vocabulary, which is public. + | Command::Quit + | Command::Client(_) + | Command::CommandQuery(_) + // CLUSTER and MODULE answer the same sentence to everyone — "not a + // cluster", "no modules" — and describe no state a scope could protect. + | Command::Cluster(_) + | Command::Module(_) + // Every MEMORY subcommand other than USAGE is refused outright, so + // there is nothing here to scope either. USAGE reads a key and is + // classified with the key commands below. + | Command::Memory(_) + | Command::Unknown(_) => CommandScope::KeyLess, + + Command::Keys(_) + | Command::Scan(_, _, _) + | Command::DbSize + | Command::FlushDb + | Command::Save + | Command::BgSave + | Command::LastSave + // INFO reports server-wide state — uptime, client counts, keyspace + // size, replication topology. A connection scoped to a handful of keys + // has no business reading it. + | Command::Info(_) + // CONFIG reports server-wide limits and whether auth is on. Same + // reasoning as INFO: not for a connection scoped to a few keys. + | Command::Config(_) + // PUBSUB enumerates every channel every other client is subscribed to. + // A scoped connection can already SUBSCRIBE to any channel it can name + // — channels are outside the scope system entirely — but naming and + // listing are different powers, the same way GET is scoped and KEYS is + // Admin. NUMSUB and NUMPAT ride along rather than splitting the family + // across two scopes for one subcommand's worth of difference. + | Command::PubSub(_) + | Command::ReplicaOfNoOne => CommandScope::Admin, + + Command::ESet(k, _) + | Command::Set(k, _, _) + | Command::Get(k) + | Command::Append(k, _) + | Command::Strlen(k) + | Command::GetRange(k, _, _) + | Command::GetSet(k, _) + | Command::SetNx(k, _) + | Command::SetEx(k, _, _) + | Command::PSetEx(k, _, _) + | Command::Incr(k) + | Command::Decr(k) + | Command::IncrBy(k, _) + | Command::DecrBy(k, _) + | Command::Expire(k, _) + | Command::PExpire(k, _) + | Command::ExpireAt(k, _) + | Command::PExpireAt(k, _) + | Command::Ttl(k) + | Command::PTtl(k) + | Command::Persist(k) + | Command::Type(k) + | Command::MemoryUsage(k) + | Command::HSet(k, _) + | Command::HGet(k, _) + | Command::HGetAll(k) + | Command::HDel(k, _) + | Command::HKeys(k) + | Command::HVals(k) + | Command::HLen(k) + | Command::HIncrBy(k, _, _) + | Command::HIncrByFloat(k, _, _) + | Command::HExists(k, _) + | Command::HSetNx(k, _, _) + | Command::HMGet(k, _) + | Command::HScan(k, _) + | Command::SScan(k, _) + | Command::ZScan(k, _) + | Command::LPush(k, _) + | Command::RPush(k, _) + | Command::LPushX(k, _) + | Command::RPushX(k, _) + | Command::LPop(k, _) + | Command::RPop(k, _) + | Command::LRange(k, _, _) + | Command::LLen(k) + | Command::LIndex(k, _) + | Command::LSet(k, _, _) + | Command::LRem(k, _, _) + | Command::LTrim(k, _, _) + | Command::SAdd(k, _) + | Command::SMembers(k) + | Command::SRem(k, _) + | Command::SCard(k) + | Command::SIsMember(k, _) + | Command::SMIsMember(k, _) + | Command::SPop(k, _) + | Command::SRandMember(k, _) + | Command::ZAdd(k, _, _) + | Command::ZRange(k, _, _, _) + | Command::ZRevRange(k, _, _, _) + | Command::ZRangeByScore(k, _, _, _, _) + | Command::ZRevRangeByScore(k, _, _, _, _) + | Command::ZScore(k, _) + | Command::ZMScore(k, _) + | Command::ZRank(k, _) + | Command::ZRevRank(k, _) + | Command::ZRem(k, _) + | Command::ZCard(k) + | Command::ZIncrBy(k, _, _) + | Command::ZCount(k, _, _) + | Command::RlSet(k, _, _) + | Command::RlCheck(k, _) + | Command::JSet(k, _, _) + | Command::JGet(k, _) + | Command::JMerge(k, _) => CommandScope::Keys(vec![k.clone()]), + + Command::Del(keys) + | Command::Unlink(keys) + | Command::MGet(keys) + | Command::Exists(keys) + | Command::SInter(keys) + | Command::SUnion(keys) + | Command::SDiff(keys) + | Command::Watch(keys) + | Command::Unwatch(keys) => CommandScope::Keys(keys.clone()), + + Command::MSet(pairs) => CommandScope::Keys(pairs.iter().map(|(k, _)| k.clone()).collect()), + Command::Rename(src, dst) | Command::SMove(src, dst, _) => { + CommandScope::Keys(vec![src.clone(), dst.clone()]) + } + Command::SInterStore(dst, keys) + | Command::SUnionStore(dst, keys) + | Command::SDiffStore(dst, keys) => { + let mut all = keys.clone(); + all.push(dst.clone()); + CommandScope::Keys(all) + } + + // Scope enforcement applies to the wrapped command. + Command::Dedup(_, _, inner) => command_scope(inner), + } +} + +/// Handle the SYNC command for one WebSocket connection, returning the RESP +/// reply. Forms: +/// `SYNC` — list this connection's current scopes +/// `SYNC TOKEN ` — set scopes from a signed token (requires +/// `RECACHED_SYNC_SECRET` on the server) +/// `SYNC [...]` — set scopes directly (only allowed when no +/// secret is configured — a bandwidth filter, not +/// an authorization boundary) +pub(crate) fn handle_sync_command( + args: &[String], + secret: Option<&str>, + scopes: &mut Option>, + conn_id: u64, +) -> Vec { + fn patterns_reply(patterns: &[String]) -> Vec { + Value::Array(Some( + patterns + .iter() + .map(|p| Value::BulkString(Some(p.clone().into_bytes()))) + .collect(), + )) + .serialize() + } + match args { + [] => patterns_reply(scopes.as_deref().unwrap_or(&[])), + [kw, token] if kw.eq_ignore_ascii_case("token") => { + let Some(secret) = secret else { + return b"-ERR SYNC TOKEN requires RECACHED_SYNC_SECRET to be configured on the server\r\n" + .to_vec(); + }; + match verify_sync_token(secret, token) { + Ok(patterns) => { + info!("WS conn {} scoped via token: {:?}", conn_id, patterns); + let reply = patterns_reply(&patterns); + *scopes = Some(patterns); + reply + } + Err(e) => Value::Error(format!("ERR invalid sync token: {}", e)).serialize(), + } + } + patterns => { + if secret.is_some() { + return b"-ERR this server requires signed scopes: use SYNC TOKEN \r\n" + .to_vec(); + } + let pats: Vec = patterns.iter().filter(|p| !p.is_empty()).cloned().collect(); + if pats.is_empty() { + return b"-ERR SYNC requires at least one pattern\r\n".to_vec(); + } + info!("WS conn {} sync scopes set: {:?}", conn_id, pats); + let reply = patterns_reply(&pats); + *scopes = Some(pats); + reply + } + } +} + +// ── connection identity ────────────────────────────────────────────────────── diff --git a/server-native/src/tests.rs b/server-native/src/tests.rs new file mode 100644 index 0000000..a5b16f3 --- /dev/null +++ b/server-native/src/tests.rs @@ -0,0 +1,5108 @@ +//! Integration tests: a real server over a real socket, plus the RespClient +//! harness the connection-level suites share. + +use crate::*; +use core_engine::cmd::{ScanArgs, SetOptions, ZAddOptions}; +use core_engine::resp::Value; +use core_engine::store::KeyValueStore; +use std::sync::atomic::{AtomicBool, AtomicI64}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::TcpStream; + +fn tmp_path(name: &str) -> PathBuf { + std::env::temp_dir().join(format!("recached_test_{name}_{}", std::process::id())) +} + +// ── TestServer harness ──────────────────────────────────────────────────── + +pub(super) struct TestServer { + pub tcp_addr: std::net::SocketAddr, + pub store: Arc, + pub state: Arc, + _task: tokio::task::JoinHandle<()>, +} + +impl Drop for TestServer { + fn drop(&mut self) { + self._task.abort(); + } +} + +pub(super) async fn spawn_server() -> TestServer { + spawn_server_cfg(None, None, false).await +} + +async fn spawn_server_cfg( + password: Option<&str>, + snap_path: Option, + start_as_replica: bool, +) -> TestServer { + let store = Arc::new(KeyValueStore::new()); + let (tx, _rx) = broadcast::channel::(256); + let pubsub: SharedPubSub = Arc::new(tokio::sync::Mutex::new(PubSubHub::new())); + let watch_registry: WatchRegistry = WatchHub::new(); + let semaphore = Arc::new(Semaphore::new(64)); + let snap_cfg = Arc::new(SnapshotConfig { + path: snap_path.unwrap_or_else(|| tmp_path("test.rdb")), + last_save: AtomicI64::new(now_unix_secs()), + }); + let state = Arc::new(ServerState { + snap: snap_cfg, + aof: None, + replicas: ReplHub::new(), + is_replica: AtomicBool::new(start_as_replica), + dedup: std::sync::Mutex::new(HashMap::new()), + ephemeral: std::sync::Mutex::new(HashMap::new()), + dedup_dirty: std::sync::atomic::AtomicBool::new(false), + }); + + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let store2 = Arc::clone(&store); + let state2 = Arc::clone(&state); + let pass = Arc::new(password.map(|s| s.to_string())); + + let task = tokio::spawn(async move { + loop { + let Ok((socket, _)) = listener.accept().await else { + return; + }; + let Ok(permit) = Arc::clone(&semaphore).try_acquire_owned() else { + continue; + }; + let (s, t, p, ps, wr, st) = ( + Arc::clone(&store2), + tx.clone(), + Arc::clone(&pass), + Arc::clone(&pubsub), + Arc::clone(&watch_registry), + Arc::clone(&state2), + ); + tokio::spawn(async move { + let peer = socket + .peer_addr() + .map(|a| a.to_string()) + .unwrap_or_default(); + handle_tcp(socket, s, t, p, ps, wr, st, peer).await; + drop(permit); + }); + } + }); + + TestServer { + tcp_addr: addr, + store, + state, + _task: task, + } +} + +// ── RespClient ──────────────────────────────────────────────────────────── + +pub(super) struct RespClient { + stream: TcpStream, + buf: Vec, + filled: usize, +} + +impl RespClient { + pub(super) async fn connect(addr: std::net::SocketAddr) -> Self { + Self { + stream: TcpStream::connect(addr).await.unwrap(), + buf: vec![0u8; 65536], + filled: 0, + } + } + + /// Send `args` and read one value. An empty `args` sends nothing and + /// just reads the next frame — used to await an out-of-band push. + pub(super) async fn cmd(&mut self, args: &[&str]) -> Value { + if !args.is_empty() { + let mut req = format!("*{}\r\n", args.len()); + for a in args { + req.push_str(&format!("${}\r\n{}\r\n", a.len(), a)); + } + self.stream.write_all(req.as_bytes()).await.unwrap(); + } + loop { + match Value::parse(&self.buf[..self.filled]) { + Ok((val, n)) => { + self.buf.copy_within(n..self.filled, 0); + self.filled -= n; + return val; + } + Err(e) if e.is_incomplete() => { + let n = self + .stream + .read(&mut self.buf[self.filled..]) + .await + .unwrap(); + assert!(n > 0, "server closed connection unexpectedly"); + self.filled += n; + } + Err(e) => panic!("RESP parse error: {e}"), + } + } + } + + /// True once the peer has closed its half of the connection. + async fn read_raw_eof(&mut self) -> bool { + let mut buf = [0u8; 64]; + matches!(self.stream.read(&mut buf).await, Ok(0)) + } + + async fn read_until_closed(&mut self) { + let mut buf = [0u8; 64]; + while self.stream.read(&mut buf).await.unwrap_or(0) > 0 {} + } +} + +fn ok() -> Value { + Value::SimpleString("OK".to_string()) +} +fn nil() -> Value { + Value::BulkString(None) +} +fn bulk(s: &str) -> Value { + Value::BulkString(Some(s.as_bytes().to_vec())) +} +fn int(n: i64) -> Value { + Value::Integer(n) +} +fn arr(items: &[&str]) -> Value { + Value::Array(Some(items.iter().map(|s| bulk(s)).collect())) +} + +// ── is_write_command ────────────────────────────────────────────────────── + +#[test] +fn is_write_command_classifies_correctly() { + assert!(is_write_command(&Command::Set( + "k".into(), + "v".into(), + SetOptions::default() + ))); + assert!(is_write_command(&Command::Del(vec!["k".into()]))); + assert!(is_write_command(&Command::Incr("k".into()))); + assert!(is_write_command(&Command::FlushDb)); + assert!(is_write_command(&Command::HSet( + "h".into(), + vec![("f".into(), "v".into())] + ))); + assert!(is_write_command(&Command::LPush( + "l".into(), + vec!["v".into()] + ))); + assert!(is_write_command(&Command::SAdd( + "s".into(), + vec!["m".into()] + ))); + assert!(is_write_command(&Command::ZAdd( + "z".into(), + ZAddOptions::default(), + vec![(1.0, "m".into())] + ))); + // reads + assert!(!is_write_command(&Command::Get("k".into()))); + assert!(!is_write_command(&Command::HGet("h".into(), "f".into()))); + assert!(!is_write_command(&Command::LRange("l".into(), 0, -1))); + assert!(!is_write_command(&Command::SMembers("s".into()))); + assert!(!is_write_command(&Command::DbSize)); + assert!(!is_write_command(&Command::Ping(None))); + assert!(!is_write_command(&Command::Publish( + "ch".into(), + "msg".into() + ))); +} + +// ── AOF replay ──────────────────────────────────────────────────────────── + +#[tokio::test] +async fn replay_aof_missing_file() { + let store = KeyValueStore::new(); + let path = tmp_path("aof_missing"); + let count = replay_aof(&store, &path).await; + assert_eq!(count, 0); +} + +#[tokio::test] +async fn replay_aof_basic() { + let store = KeyValueStore::new(); + let path = tmp_path("aof_basic.aof"); + let resp = "*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$3\r\nbar\r\n\ + *3\r\n$3\r\nSET\r\n$3\r\nbaz\r\n$3\r\nqux\r\n"; + tokio::fs::write(&path, resp.as_bytes()).await.unwrap(); + let count = replay_aof(&store, &path).await; + assert_eq!(count, 2); + assert_eq!(store.execute(Command::DbSize), Value::Integer(2)); + let _ = tokio::fs::remove_file(&path).await; +} + +#[tokio::test] +async fn replay_aof_push_frames() { + // The live server records writes via `on_write`, which stores them in + // RESP3 Push (`>`) form. Replay must accept those, not just `*` arrays. + let store = KeyValueStore::new(); + let path = tmp_path("aof_push.aof"); + let resp = ">3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$3\r\nbar\r\n"; + tokio::fs::write(&path, resp.as_bytes()).await.unwrap(); + let count = replay_aof(&store, &path).await; + assert_eq!(count, 1); + assert_eq!( + store.execute(Command::Get("foo".into())), + Value::BulkString(Some(b"bar".to_vec())) + ); + let _ = tokio::fs::remove_file(&path).await; +} + +// ── Snapshot save / load ────────────────────────────────────────────────── + +#[tokio::test] +async fn snapshot_save_and_load() { + let store = KeyValueStore::new(); + store.execute(Command::Set( + "hello".into(), + "world".into(), + SetOptions::default(), + )); + let path = tmp_path("snap.rdb"); + let cfg = Arc::new(SnapshotConfig { + path: path.clone(), + last_save: AtomicI64::new(0), + }); + save_snapshot(&store, &cfg).await; + assert!(path.exists()); + let store2 = KeyValueStore::new(); + let loaded = load_snapshot(&store2, &path).await; + assert!(loaded); + assert_eq!( + store2.execute(Command::Get("hello".into())), + Value::BulkString(Some(b"world".to_vec())) + ); + let _ = tokio::fs::remove_file(&path).await; +} + +// ── AofWriter append / truncate ─────────────────────────────────────────── + +#[tokio::test] +async fn aof_writer_append_and_truncate() { + let path = tmp_path("aof_writer.aof"); + let aof = AofWriter::open(path.clone(), AofSync::No).await.unwrap(); + aof.append(b"*3\r\n$3\r\nSET\r\n$1\r\nk\r\n$1\r\nv\r\n") + .await; + aof.flush().await; + let len_before = tokio::fs::metadata(&path).await.unwrap().len(); + assert!(len_before > 0); + aof.truncate().await; + let len_after = tokio::fs::metadata(&path).await.unwrap().len(); + assert_eq!(len_after, 0); + let _ = tokio::fs::remove_file(&path).await; +} + +// ── Integration: 3a basic commands ─────────────────────────────────────── + +#[tokio::test] +async fn integration_set_get_del() { + let srv = spawn_server().await; + let mut c = RespClient::connect(srv.tcp_addr).await; + + assert_eq!(c.cmd(&["SET", "k", "v"]).await, ok()); + assert_eq!(c.cmd(&["GET", "k"]).await, bulk("v")); + assert_eq!(c.cmd(&["GET", "missing"]).await, nil()); + assert_eq!(c.cmd(&["DEL", "k"]).await, int(1)); + assert_eq!(c.cmd(&["GET", "k"]).await, nil()); + assert_eq!(c.cmd(&["DEL", "k"]).await, int(0)); // already gone +} + +#[tokio::test] +async fn integration_binary_value_round_trips_over_resp() { + // The drop-in claim runs through this port: a value that is not valid + // UTF-8 must come back byte-for-byte, exactly as Redis would. + let srv = spawn_server().await; + let mut c = RespClient::connect(srv.tcp_addr).await; + + let binary: &[u8] = &[0xff, 0xfe, 0x00, 0x41, 0x80]; + let mut req = b"*3\r\n$3\r\nSET\r\n$3\r\nbin\r\n".to_vec(); + req.extend_from_slice(format!("${}\r\n", binary.len()).as_bytes()); + req.extend_from_slice(binary); + req.extend_from_slice(b"\r\n"); + c.stream.write_all(&req).await.unwrap(); + assert_eq!(c.cmd(&[]).await, ok()); + + assert_eq!( + c.cmd(&["GET", "bin"]).await, + Value::BulkString(Some(binary.to_vec())), + "binary value must survive the round trip" + ); + assert_eq!( + c.cmd(&["STRLEN", "bin"]).await, + int(binary.len() as i64), + "length is counted in bytes" + ); +} + +#[tokio::test] +async fn integration_binary_key_is_refused_over_resp() { + // Keys stay text: they are glob-matched and scope-checked, so a + // corrupted one would be silently unreachable. The refusal must be a + // clean RESP error that leaves the connection usable. + let srv = spawn_server().await; + let mut c = RespClient::connect(srv.tcp_addr).await; + + c.stream + .write_all(b"*3\r\n$3\r\nSET\r\n$2\r\n\xff\xfe\r\n$1\r\nv\r\n") + .await + .unwrap(); + let reply = c.cmd(&[]).await; + let Value::Error(e) = &reply else { + panic!("binary key must be refused, got {reply:?}") + }; + assert!(e.contains("must be text"), "error must explain: {e:?}"); + + assert_eq!(c.cmd(&["DBSIZE"]).await, int(0), "nothing may be stored"); + assert_eq!(c.cmd(&["SET", "ok", "v"]).await, ok()); +} + +#[tokio::test] +async fn integration_hello_negotiates_the_protocol() { + let srv = spawn_server().await; + let mut c = RespClient::connect(srv.tcp_addr).await; + + // Default is RESP2: the reply is a flat array, not a map. + let v = c.cmd(&["HELLO"]).await; + let Value::Array(Some(items)) = v else { + panic!("RESP2 HELLO must reply with an array, got {v:?}") + }; + assert!(items.contains(&bulk("recached"))); + assert!(items.contains(&Value::Integer(2))); + + // Upgrading yields a map keyed the same way. + let v = c.cmd(&["HELLO", "3"]).await; + let Value::Map(pairs) = v else { + panic!("RESP3 HELLO must reply with a map, got {v:?}") + }; + let proto = pairs + .iter() + .find(|(k, _)| *k == bulk("proto")) + .map(|(_, v)| v.clone()); + assert_eq!(proto, Some(Value::Integer(3))); + + // An unsupported version is refused and the connection stays usable. + let v = c.cmd(&["HELLO", "9"]).await; + assert!( + matches!(&v, Value::Error(e) if e.starts_with("NOPROTO")), + "expected NOPROTO, got {v:?}" + ); + assert_eq!(c.cmd(&["PING"]).await, Value::SimpleString("PONG".into())); +} + +#[tokio::test] +async fn integration_pubsub_frame_type_follows_the_negotiated_protocol() { + // The bug this pins: pub/sub deliveries were RESP3 push frames on every + // connection, including RESP2 ones that cannot parse `>` at all. + for (protover, want_push) in [(None, false), (Some("3"), true)] { + let srv = spawn_server().await; + let mut sub = RespClient::connect(srv.tcp_addr).await; + if let Some(v) = protover { + sub.cmd(&["HELLO", v]).await; + } + assert!(matches!( + sub.cmd(&["SUBSCRIBE", "news"]).await, + Value::Array(_) | Value::Push(_) + )); + + let mut pubr = RespClient::connect(srv.tcp_addr).await; + // Wait for the subscription to register before publishing. + for _ in 0..50 { + if pubr.cmd(&["PUBLISH", "news", "hi"]).await == int(1) { + break; + } + tokio::time::sleep(tokio::time::Duration::from_millis(20)).await; + } + + let delivery = sub.cmd(&[]).await; + match (&delivery, want_push) { + (Value::Push(_), true) => {} + (Value::Array(Some(_)), false) => {} + _ => panic!("protover {protover:?}: expected push={want_push}, got {delivery:?}"), + } + } +} + +#[tokio::test] +async fn integration_incr_and_expiry() { + let srv = spawn_server().await; + let mut c = RespClient::connect(srv.tcp_addr).await; + + assert_eq!(c.cmd(&["SET", "n", "10"]).await, ok()); + assert_eq!(c.cmd(&["INCR", "n"]).await, int(11)); + assert_eq!(c.cmd(&["INCRBY", "n", "4"]).await, int(15)); + assert_eq!(c.cmd(&["DECR", "n"]).await, int(14)); + + // TTL: set a key with 1-second expiry and verify TTL and eventual expiry + assert_eq!(c.cmd(&["SET", "ex", "val", "EX", "1"]).await, ok()); + let ttl = c.cmd(&["TTL", "ex"]).await; + assert!(matches!(ttl, Value::Integer(1) | Value::Integer(0))); + tokio::time::sleep(tokio::time::Duration::from_millis(1100)).await; + assert_eq!(c.cmd(&["GET", "ex"]).await, nil()); +} + +#[tokio::test] +async fn integration_string_commands() { + let srv = spawn_server().await; + let mut c = RespClient::connect(srv.tcp_addr).await; + + // APPEND + STRLEN + assert_eq!(c.cmd(&["APPEND", "s", "hello"]).await, int(5)); + assert_eq!(c.cmd(&["APPEND", "s", " world"]).await, int(11)); + assert_eq!(c.cmd(&["STRLEN", "s"]).await, int(11)); + + // GETSET + assert_eq!(c.cmd(&["GETSET", "s", "new"]).await, bulk("hello world")); + assert_eq!(c.cmd(&["GET", "s"]).await, bulk("new")); + + // SETNX + assert_eq!(c.cmd(&["SETNX", "nx", "first"]).await, int(1)); + assert_eq!(c.cmd(&["SETNX", "nx", "second"]).await, int(0)); + assert_eq!(c.cmd(&["GET", "nx"]).await, bulk("first")); + + // SETEX + assert_eq!(c.cmd(&["SETEX", "ex", "60", "val"]).await, ok()); + let ttl = c.cmd(&["TTL", "ex"]).await; + assert!(matches!(ttl, Value::Integer(t) if t > 0 && t <= 60)); + + // MSET / MGET + assert_eq!(c.cmd(&["MSET", "a", "1", "b", "2", "c", "3"]).await, ok()); + let got = c.cmd(&["MGET", "a", "b", "c", "missing"]).await; + assert_eq!( + got, + Value::Array(Some(vec![bulk("1"), bulk("2"), bulk("3"), nil()])) + ); +} + +#[tokio::test] +async fn integration_bounded_reads_over_resp() { + // The pair of primitives a client needs to inspect a large key without + // pulling it whole: a byte window into a string, and a cursor over a + // collection. Exercised over the wire because that is where the reply + // shape — bulk cursor, nested array — has to be right. + let srv = spawn_server().await; + let mut c = RespClient::connect(srv.tcp_addr).await; + + c.cmd(&["SET", "s", "This is a string"]).await; + assert_eq!(c.cmd(&["GETRANGE", "s", "0", "3"]).await, bulk("This")); + assert_eq!(c.cmd(&["GETRANGE", "s", "-6", "-1"]).await, bulk("string")); + assert_eq!(c.cmd(&["GETRANGE", "ghost", "0", "-1"]).await, bulk("")); + + c.cmd(&["HSET", "h", "a", "1", "b", "2", "c", "3"]).await; + assert_eq!( + c.cmd(&["HSCAN", "h", "0", "COUNT", "2"]).await, + Value::Array(Some(vec![ + bulk("2"), + Value::Array(Some(vec![bulk("a"), bulk("1"), bulk("b"), bulk("2")])), + ])) + ); + assert_eq!( + c.cmd(&["HSCAN", "h", "2"]).await, + Value::Array(Some(vec![ + bulk("0"), + Value::Array(Some(vec![bulk("c"), bulk("3")])), + ])) + ); + assert_eq!( + c.cmd(&["HSCAN", "h", "0", "NOVALUES"]).await, + Value::Array(Some(vec![ + bulk("0"), + Value::Array(Some(vec![bulk("a"), bulk("b"), bulk("c")])), + ])) + ); + + c.cmd(&["SADD", "st", "x", "y"]).await; + assert_eq!( + c.cmd(&["SSCAN", "st", "0", "MATCH", "x*"]).await, + Value::Array(Some(vec![bulk("0"), Value::Array(Some(vec![bulk("x")])),])) + ); + + c.cmd(&["ZADD", "z", "1.5", "amy"]).await; + assert_eq!( + c.cmd(&["ZSCAN", "z", "0"]).await, + Value::Array(Some(vec![ + bulk("0"), + Value::Array(Some(vec![bulk("amy"), bulk("1.5")])), + ])) + ); +} + +#[tokio::test] +async fn integration_bounded_reads_are_allowed_on_a_replica() { + // Read-only by construction: a replica must serve them, and the + // is_write_command allowlist is what decides that. + let srv = spawn_server_cfg(None, None, true).await; + let mut c = RespClient::connect(srv.tcp_addr).await; + + assert_eq!(c.cmd(&["GETRANGE", "s", "0", "-1"]).await, bulk("")); + assert_eq!( + c.cmd(&["HSCAN", "h", "0"]).await, + Value::Array(Some(vec![bulk("0"), Value::Array(Some(vec![]))])) + ); +} + +#[tokio::test] +async fn integration_handshake_commands_over_resp() { + // Every current client library opens with HELLO + CLIENT SETINFO and + // closes with QUIT. This is that sequence, on the wire. + let srv = spawn_server().await; + let mut c = RespClient::connect(srv.tcp_addr).await; + + assert_eq!( + c.cmd(&["CLIENT", "SETINFO", "LIB-NAME", "node-redis"]) + .await, + ok() + ); + assert_eq!( + c.cmd(&["CLIENT", "SETINFO", "LIB-VER", "6.2.0"]).await, + ok() + ); + assert_eq!(c.cmd(&["CLIENT", "SETNAME", "tap"]).await, ok()); + assert_eq!(c.cmd(&["CLIENT", "GETNAME"]).await, bulk("tap")); + + let Value::Integer(id) = c.cmd(&["CLIENT", "ID"]).await else { + panic!("CLIENT ID must be an integer") + }; + assert!(id > 0); + + let Value::BulkString(Some(info)) = c.cmd(&["CLIENT", "INFO"]).await else { + panic!("CLIENT INFO must be a bulk string") + }; + let info = String::from_utf8(info).unwrap(); + assert!(info.contains(&format!("id={id}")), "{info}"); + assert!(info.contains("lib-name=node-redis"), "{info}"); + assert!(info.contains("name=tap"), "{info}"); + assert!(info.contains("addr=127.0.0.1:"), "{info}"); + + // This connection must appear in the list it asks for. + let Value::BulkString(Some(list)) = c.cmd(&["CLIENT", "LIST"]).await else { + panic!("CLIENT LIST must be a bulk string") + }; + let list = String::from_utf8(list).unwrap(); + assert!( + list.lines().any(|l| l.contains(&format!("id={id}"))), + "{list}" + ); + + assert_eq!( + c.cmd(&["CONFIG", "GET", "maxmemory-policy"]).await, + Value::Array(Some(vec![bulk("maxmemory-policy"), bulk("noeviction")])) + ); + + let Value::Integer(n) = c.cmd(&["COMMAND", "COUNT"]).await else { + panic!("COMMAND COUNT must be an integer") + }; + assert!( + n > 100, + "the catalog should cover the whole command set, got {n}" + ); +} + +#[tokio::test] +async fn integration_quit_replies_then_closes() { + let srv = spawn_server().await; + let mut c = RespClient::connect(srv.tcp_addr).await; + assert_eq!(c.cmd(&["PING"]).await, Value::SimpleString("PONG".into())); + // +OK first, then the close — a client that reads its reply before + // dropping the socket must not see a connection error instead. + assert_eq!(c.cmd(&["QUIT"]).await, ok()); + assert!( + c.read_raw_eof().await, + "the server must close the connection after QUIT" + ); +} + +#[tokio::test] +async fn integration_quit_works_before_authentication() { + // Redis flags QUIT no_auth. A client that cannot authenticate still + // gets a clean close instead of leaving a socket parked on the server. + let srv = spawn_server_cfg(Some("hunter2"), None, false).await; + let mut c = RespClient::connect(srv.tcp_addr).await; + assert!(matches!(c.cmd(&["PING"]).await, Value::Error(e) if e.contains("NOAUTH"))); + assert_eq!(c.cmd(&["QUIT"]).await, ok()); +} + +#[tokio::test] +async fn integration_client_list_sees_other_connections() { + let srv = spawn_server().await; + let mut a = RespClient::connect(srv.tcp_addr).await; + let mut b = RespClient::connect(srv.tcp_addr).await; + b.cmd(&["CLIENT", "SETNAME", "second"]).await; + + let Value::BulkString(Some(list)) = a.cmd(&["CLIENT", "LIST"]).await else { + panic!("expected a bulk string") + }; + let list = String::from_utf8(list).unwrap(); + assert!( + list.lines().any(|l| l.contains("name=second")), + "a connection must see its peers, got:\n{list}" + ); + assert!(list.lines().count() >= 2, "{list}"); +} + +#[tokio::test] +async fn integration_hash_commands() { + let srv = spawn_server().await; + let mut c = RespClient::connect(srv.tcp_addr).await; + + assert_eq!(c.cmd(&["HSET", "h", "f1", "v1", "f2", "v2"]).await, int(2)); + assert_eq!(c.cmd(&["HGET", "h", "f1"]).await, bulk("v1")); + assert_eq!(c.cmd(&["HGET", "h", "missing"]).await, nil()); + assert_eq!(c.cmd(&["HLEN", "h"]).await, int(2)); + assert_eq!(c.cmd(&["HDEL", "h", "f1"]).await, int(1)); + assert_eq!(c.cmd(&["HLEN", "h"]).await, int(1)); + // HGETALL returns field-value pairs + let all = c.cmd(&["HGETALL", "h"]).await; + assert_eq!(all, Value::Array(Some(vec![bulk("f2"), bulk("v2")]))); +} + +#[tokio::test] +async fn integration_list_commands() { + let srv = spawn_server().await; + let mut c = RespClient::connect(srv.tcp_addr).await; + + assert_eq!(c.cmd(&["RPUSH", "l", "a", "b", "c"]).await, int(3)); + assert_eq!(c.cmd(&["LPUSH", "l", "z"]).await, int(4)); + assert_eq!(c.cmd(&["LLEN", "l"]).await, int(4)); + assert_eq!( + c.cmd(&["LRANGE", "l", "0", "-1"]).await, + Value::Array(Some(vec![bulk("z"), bulk("a"), bulk("b"), bulk("c")])) + ); + assert_eq!(c.cmd(&["LPOP", "l"]).await, bulk("z")); + assert_eq!(c.cmd(&["RPOP", "l"]).await, bulk("c")); + assert_eq!(c.cmd(&["LLEN", "l"]).await, int(2)); +} + +#[tokio::test] +async fn integration_set_commands() { + let srv = spawn_server().await; + let mut c = RespClient::connect(srv.tcp_addr).await; + + assert_eq!(c.cmd(&["SADD", "s", "a", "b", "c"]).await, int(3)); + assert_eq!(c.cmd(&["SADD", "s", "a"]).await, int(0)); // duplicate + assert_eq!(c.cmd(&["SCARD", "s"]).await, int(3)); + assert_eq!(c.cmd(&["SISMEMBER", "s", "b"]).await, int(1)); + assert_eq!(c.cmd(&["SISMEMBER", "s", "x"]).await, int(0)); + assert_eq!(c.cmd(&["SREM", "s", "a"]).await, int(1)); + assert_eq!(c.cmd(&["SCARD", "s"]).await, int(2)); +} + +#[tokio::test] +async fn integration_zset_commands() { + let srv = spawn_server().await; + let mut c = RespClient::connect(srv.tcp_addr).await; + + assert_eq!( + c.cmd(&["ZADD", "z", "1.5", "a", "2.5", "b", "3.0", "c"]) + .await, + int(3) + ); + assert_eq!(c.cmd(&["ZCARD", "z"]).await, int(3)); + assert_eq!(c.cmd(&["ZSCORE", "z", "b"]).await, bulk("2.5")); + assert_eq!(c.cmd(&["ZRANK", "z", "a"]).await, int(0)); + assert_eq!(c.cmd(&["ZRANK", "z", "c"]).await, int(2)); + assert_eq!( + c.cmd(&["ZRANGE", "z", "0", "-1", "WITHSCORES"]).await, + Value::Array(Some(vec![ + bulk("a"), + bulk("1.5"), + bulk("b"), + bulk("2.5"), + bulk("c"), + bulk("3"), + ])) + ); + assert_eq!(c.cmd(&["ZREM", "z", "b"]).await, int(1)); + assert_eq!(c.cmd(&["ZCARD", "z"]).await, int(2)); +} + +#[tokio::test] +async fn integration_transactions_exec() { + let srv = spawn_server().await; + let mut c = RespClient::connect(srv.tcp_addr).await; + + assert_eq!(c.cmd(&["SET", "counter", "10"]).await, ok()); + assert_eq!(c.cmd(&["MULTI"]).await, ok()); + assert_eq!( + c.cmd(&["SET", "counter", "20"]).await, + Value::SimpleString("QUEUED".to_string()) + ); + assert_eq!( + c.cmd(&["INCR", "counter"]).await, + Value::SimpleString("QUEUED".to_string()) + ); + let res = c.cmd(&["EXEC"]).await; + assert_eq!(res, Value::Array(Some(vec![ok(), int(21)]))); + assert_eq!(c.cmd(&["GET", "counter"]).await, bulk("21")); +} + +#[tokio::test] +async fn integration_transactions_discard() { + let srv = spawn_server().await; + let mut c = RespClient::connect(srv.tcp_addr).await; + + assert_eq!(c.cmd(&["SET", "key", "original"]).await, ok()); + assert_eq!(c.cmd(&["MULTI"]).await, ok()); + assert_eq!( + c.cmd(&["DEL", "key"]).await, + Value::SimpleString("QUEUED".to_string()) + ); + assert_eq!(c.cmd(&["DISCARD"]).await, ok()); + assert_eq!(c.cmd(&["GET", "key"]).await, bulk("original")); // DEL was discarded +} + +#[tokio::test] +async fn integration_unknown_command() { + let srv = spawn_server().await; + let mut c = RespClient::connect(srv.tcp_addr).await; + + let r = c.cmd(&["NOTACOMMAND", "arg"]).await; + assert!(matches!(r, Value::Error(_))); +} + +// ── Integration: 3b auth ────────────────────────────────────────────────── + +#[tokio::test] +async fn integration_auth_blocks_unauthenticated() { + let srv = spawn_server_cfg(Some("secret"), None, false).await; + let mut c = RespClient::connect(srv.tcp_addr).await; + + let r = c.cmd(&["SET", "k", "v"]).await; + assert!(matches!(&r, Value::Error(e) if e.contains("NOAUTH"))); +} + +#[tokio::test] +async fn integration_auth_correct() { + let srv = spawn_server_cfg(Some("secret"), None, false).await; + let mut c = RespClient::connect(srv.tcp_addr).await; + + assert_eq!(c.cmd(&["AUTH", "secret"]).await, ok()); + assert_eq!(c.cmd(&["SET", "k", "v"]).await, ok()); + assert_eq!(c.cmd(&["GET", "k"]).await, bulk("v")); +} + +#[tokio::test] +async fn integration_auth_wrong_password_lockout() { + let srv = spawn_server_cfg(Some("secret"), None, false).await; + let mut c = RespClient::connect(srv.tcp_addr).await; + + // First 4 wrong attempts → "ERR invalid password" + for _ in 0..4 { + let r = c.cmd(&["AUTH", "wrong"]).await; + assert!(matches!(&r, Value::Error(e) if e.contains("invalid"))); + } + // 5th attempt hits MAX_AUTH_FAILURES → "too many" + server disconnects + let r = c.cmd(&["AUTH", "wrong"]).await; + assert!(matches!(&r, Value::Error(e) if e.contains("too many"))); + c.read_until_closed().await; +} + +// ── Integration: 3c persistence ─────────────────────────────────────────── + +#[tokio::test] +async fn integration_save_and_reload() { + let snap = tmp_path("integ_snap.rdb"); + let srv = spawn_server_cfg(None, Some(snap.clone()), false).await; + let mut c = RespClient::connect(srv.tcp_addr).await; + + assert_eq!(c.cmd(&["SET", "hello", "world"]).await, ok()); + assert_eq!(c.cmd(&["SET", "foo", "bar"]).await, ok()); + assert_eq!(c.cmd(&["SAVE"]).await, ok()); + + // Load into a fresh store + let store2 = KeyValueStore::new(); + let loaded = load_snapshot(&store2, &snap).await; + assert!(loaded); + assert_eq!( + store2.execute(Command::Get("hello".into())), + Value::BulkString(Some(b"world".to_vec())) + ); + assert_eq!( + store2.execute(Command::Get("foo".into())), + Value::BulkString(Some(b"bar".to_vec())) + ); + let _ = tokio::fs::remove_file(&snap).await; +} + +#[tokio::test] +async fn integration_aof_replay() { + let path = tmp_path("integ_aof.aof"); + let aof = AofWriter::open(path.clone(), AofSync::No).await.unwrap(); + let store = KeyValueStore::new(); + let snap_cfg = Arc::new(SnapshotConfig { + path: tmp_path("integ_aof.rdb"), + last_save: AtomicI64::new(0), + }); + let state = Arc::new(ServerState { + snap: snap_cfg, + aof: Some(Arc::new(aof)), + replicas: ReplHub::new(), + is_replica: AtomicBool::new(false), + dedup: std::sync::Mutex::new(HashMap::new()), + ephemeral: std::sync::Mutex::new(HashMap::new()), + dedup_dirty: std::sync::atomic::AtomicBool::new(false), + }); + + // Simulate writes captured by AOF + state + .on_write(b"*3\r\n$3\r\nSET\r\n$5\r\nhello\r\n$5\r\nworld\r\n") + .await; + state + .on_write(b"*3\r\n$3\r\nSET\r\n$3\r\nfoo\r\n$3\r\nbar\r\n") + .await; + if let Some(ref a) = state.aof { + a.flush().await; + } + + // Replay into fresh store + let store2 = KeyValueStore::new(); + let count = replay_aof(&store2, &path).await; + assert_eq!(count, 2); + assert_eq!( + store2.execute(Command::Get("hello".into())), + Value::BulkString(Some(b"world".to_vec())) + ); + drop(store); // suppress unused warning + let _ = tokio::fs::remove_file(&path).await; +} + +#[tokio::test] +async fn integration_dirty_counter() { + let srv = spawn_server().await; + let mut c = RespClient::connect(srv.tcp_addr).await; + + assert_eq!(srv.store.dirty_count(), 0); + + assert_eq!(c.cmd(&["SET", "a", "1"]).await, ok()); + assert_eq!(c.cmd(&["SET", "b", "2"]).await, ok()); + assert_eq!(srv.store.dirty_count(), 2); + + // Trigger a save — dirty resets to 0 + assert_eq!(c.cmd(&["SAVE"]).await, ok()); + assert_eq!(srv.store.dirty_count(), 0); + + // Baseline *after* the explicit save, not before it: SAVE writes + // last_save itself, so a baseline taken beforehand differs by one + // whenever the save lands in the next whole second — which is what this + // test used to fail on, roughly one run in thirty. The assertion below + // is about no *further* save happening. + let last_save = srv.state.snap.last_save.load(Ordering::Relaxed); + + // No new writes → save condition not met → last_save unchanged after 1s + tokio::time::sleep(tokio::time::Duration::from_millis(1100)).await; + assert_eq!( + last_save, + srv.state.snap.last_save.load(Ordering::Relaxed), + "no autosave should fire with no conditions configured" + ); +} + +// ── Integration: 3d replication ─────────────────────────────────────────── + +#[tokio::test] +async fn integration_replica_rejects_writes() { + let srv = spawn_server_cfg(None, None, true).await; + let mut c = RespClient::connect(srv.tcp_addr).await; + + let r = c.cmd(&["SET", "k", "v"]).await; + assert!(matches!(&r, Value::Error(e) if e.contains("READONLY"))); + // Reads still work + assert_eq!(c.cmd(&["GET", "k"]).await, nil()); +} + +#[tokio::test] +async fn integration_replicaof_no_one_promotes() { + let srv = spawn_server_cfg(None, None, true).await; + let mut c = RespClient::connect(srv.tcp_addr).await; + + // Promote + assert_eq!(c.cmd(&["REPLICAOF", "NO", "ONE"]).await, ok()); + // Now writes are accepted + assert_eq!(c.cmd(&["SET", "k", "v"]).await, ok()); + assert_eq!(c.cmd(&["GET", "k"]).await, bulk("v")); + assert!(!srv.state.is_replica()); +} + +// ── FLUSHDB reaches live queries ────────────────────────────────────────── + +/// Collect every frame arriving within `ms`, so a test can assert on the +/// keychange among the command-replay pushes that travel alongside it. +async fn drain_frames(c: &mut WsClient, ms: u64) -> Vec { + let mut out = Vec::new(); + while let Some(f) = c.recv_any(ms).await { + out.push(f); + } + out +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn flushdb_notifies_live_query_subscribers() { + // Previously FLUSHDB emitted nothing to live queries: primary_keys() is + // empty for it, so subscribers kept serving data the server had wiped. + let srv = spawn_ws_server().await; + let mut watcher = WsClient::connect(srv.tcp_addr).await; + watcher.cmd(&["QSUB", "cart:*"]).await; + + let mut writer = WsClient::connect(srv.tcp_addr).await; + writer.cmd(&["SET", "cart:item:1", "a"]).await; + drain_frames(&mut watcher, 400).await; + + writer.cmd(&["FLUSHDB"]).await; + + let frames = drain_frames(&mut watcher, 800).await; + assert!( + frames + .iter() + .any(|f| f.contains("keychange") && f.contains("cart:*")), + "expected a keychange sentinel naming the pattern, got: {frames:?}" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn flushdb_sends_one_sentinel_per_pattern_not_per_key() { + // The reason for a sentinel: announcing per key would be one frame per + // key in the keyspace for a single command. + let srv = spawn_ws_server().await; + let mut watcher = WsClient::connect(srv.tcp_addr).await; + watcher.cmd(&["QSUB", "bulk:*"]).await; + + let mut writer = WsClient::connect(srv.tcp_addr).await; + for i in 0..25 { + writer.cmd(&["SET", &format!("bulk:{i}"), "v"]).await; + } + drain_frames(&mut watcher, 500).await; + + writer.cmd(&["FLUSHDB"]).await; + + let keychanges: Vec = drain_frames(&mut watcher, 800) + .await + .into_iter() + .filter(|f| f.contains("keychange")) + .collect(); + assert_eq!( + keychanges.len(), + 1, + "25 keys wiped must produce one sentinel, not 25 frames: {keychanges:?}" + ); + assert!(keychanges[0].contains("bulk:*")); +} + +// ── Exactly-once across a restart ───────────────────────────────────────── + +/// Build a `ServerState` whose snapshot path (and therefore dedup sidecar) +/// is `path` — the same file a restarted process would find. +fn state_with_snapshot_path(path: PathBuf) -> Arc { + Arc::new(ServerState { + snap: Arc::new(SnapshotConfig { + path, + last_save: AtomicI64::new(now_unix_secs()), + }), + aof: None, + replicas: ReplHub::new(), + is_replica: AtomicBool::new(false), + dedup: std::sync::Mutex::new(HashMap::new()), + ephemeral: std::sync::Mutex::new(HashMap::new()), + dedup_dirty: std::sync::atomic::AtomicBool::new(false), + }) +} + +#[tokio::test] +async fn dedup_marks_survive_a_restart() { + let snap = tmp_path("dedup_restart.rdb"); + let _ = std::fs::remove_file(snap.with_extension("dedup")); + + // First run: the client's writes are accepted once each. + let first = state_with_snapshot_path(snap.clone()); + assert!(!first.dedup_seen("client-a", 1)); + assert!(!first.dedup_seen("client-a", 2)); + assert!( + first.dedup_seen("client-a", 2), + "same id twice within a run" + ); + first.persist_dedup().await; + + // Restart: fresh process, same snapshot path. + let second = state_with_snapshot_path(snap.clone()); + assert!( + !second.dedup_seen("client-a", 3), + "a genuinely new id must still be accepted" + ); + + let third = state_with_snapshot_path(snap.clone()); + third.load_dedup().await; + assert!( + third.dedup_seen("client-a", 2), + "a replayed write must be recognised after a restart — this is the \ + caveat the sidecar exists to close" + ); + assert!(!third.dedup_seen("client-a", 99), "higher ids still apply"); + + let _ = std::fs::remove_file(snap.with_extension("dedup")); +} + +#[tokio::test] +async fn persist_dedup_is_a_no_op_when_nothing_advanced() { + // The flusher runs every second; it must not rewrite the file when no + // mark moved. + let snap = tmp_path("dedup_noop.rdb"); + let side = snap.with_extension("dedup"); + let _ = std::fs::remove_file(&side); + + let state = state_with_snapshot_path(snap.clone()); + state.dedup_seen("c", 1); + state.persist_dedup().await; + assert!(side.exists(), "first flush should write"); + + let before = std::fs::metadata(&side).unwrap().modified().unwrap(); + state.persist_dedup().await; // nothing changed since + let after = std::fs::metadata(&side).unwrap().modified().unwrap(); + assert_eq!(before, after, "unchanged marks must not rewrite the file"); + + let _ = std::fs::remove_file(&side); +} + +#[tokio::test] +async fn a_corrupt_dedup_sidecar_is_ignored_not_fatal() { + // Losing exactly-once bookkeeping is bad; refusing to boot is worse. + let snap = tmp_path("dedup_corrupt.rdb"); + let side = snap.with_extension("dedup"); + std::fs::write(&side, b"not messagepack").unwrap(); + + let state = state_with_snapshot_path(snap.clone()); + state.load_dedup().await; // must not panic + assert!(!state.dedup_seen("client-a", 1), "server still functions"); + + let _ = std::fs::remove_file(&side); +} + +#[tokio::test] +async fn a_missing_dedup_sidecar_is_a_clean_first_boot() { + let snap = tmp_path("dedup_absent.rdb"); + let _ = std::fs::remove_file(snap.with_extension("dedup")); + let state = state_with_snapshot_path(snap); + state.load_dedup().await; + assert!(!state.dedup_seen("fresh", 1)); +} + +// ── Presence: connection-scoped keys ────────────────────────────────────── + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn eset_key_is_deleted_when_its_connection_closes() { + let srv = spawn_ws_server().await; + let mut watcher = WsClient::connect(srv.tcp_addr).await; + watcher.cmd(&["QSUB", "presence:*"]).await; + + { + let mut presence = WsClient::connect(srv.tcp_addr).await; + assert_eq!( + presence.cmd(&["ESET", "presence:user:42", "online"]).await, + Value::SimpleString("OK".into()) + ); + // Visible to everyone while the connection is open. + assert_eq!( + srv.store.execute(Command::Get("presence:user:42".into())), + Value::BulkString(Some(b"online".to_vec())) + ); + } // connection dropped here + + // The key goes away on its own — no heartbeat, no TTL to wait out. + for _ in 0..40 { + if srv.store.execute(Command::Get("presence:user:42".into())) == Value::BulkString(None) { + return; + } + tokio::time::sleep(tokio::time::Duration::from_millis(50)).await; + } + panic!("ephemeral key outlived its connection"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn eset_deletion_is_broadcast_to_live_queries() { + // Presence is only useful if peers are *told*; polling for the absence + // of a key is the thing this replaces. + let srv = spawn_ws_server().await; + let mut watcher = WsClient::connect(srv.tcp_addr).await; + watcher.cmd(&["QSUB", "presence:*"]).await; + + { + let mut presence = WsClient::connect(srv.tcp_addr).await; + presence.cmd(&["ESET", "presence:user:7", "online"]).await; + // Drain the set notification. + let _ = watcher.recv_push(1000).await; + } + + let frames = drain_frames(&mut watcher, 1500).await; + assert!( + frames + .iter() + .any(|f| f.contains("keychange") && f.contains("presence:user:7")), + "a live query must receive a keychange for the departing peer, got: {frames:?}" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_second_tab_keeps_presence_alive_when_the_first_closes() { + // The multi-tab case. Ownership transfers to the most recent writer, so + // closing an older tab must not mark the user offline. + let srv = spawn_ws_server().await; + + let mut tab_b = WsClient::connect(srv.tcp_addr).await; + { + let mut tab_a = WsClient::connect(srv.tcp_addr).await; + tab_a.cmd(&["ESET", "presence:user:9", "online"]).await; + // Tab B claims the same key — it is now the owner. + tab_b.cmd(&["ESET", "presence:user:9", "online"]).await; + } // tab A closes + + // Give the close handler time to run, then confirm the key survived. + tokio::time::sleep(tokio::time::Duration::from_millis(300)).await; + assert_eq!( + srv.store.execute(Command::Get("presence:user:9".into())), + Value::BulkString(Some(b"online".to_vec())), + "closing an older tab must not clear presence held by a newer one" + ); + + drop(tab_b); + for _ in 0..40 { + if srv.store.execute(Command::Get("presence:user:9".into())) == Value::BulkString(None) { + return; + } + tokio::time::sleep(tokio::time::Duration::from_millis(50)).await; + } + panic!("key outlived its last owner"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn plain_set_is_not_ephemeral() { + // Only ESET opts into connection-scoped lifetime; SET must be unaffected. + let srv = spawn_ws_server().await; + { + let mut c = WsClient::connect(srv.tcp_addr).await; + c.cmd(&["SET", "durable:key", "value"]).await; + } + tokio::time::sleep(tokio::time::Duration::from_millis(300)).await; + assert_eq!( + srv.store.execute(Command::Get("durable:key".into())), + Value::BulkString(Some(b"value".to_vec())) + ); +} + +#[tokio::test] +async fn integration_replica_receives_write() { + // Spawn primary with a separate replication listener on a random port + let primary = spawn_server().await; + let repl_registry: ReplRegistry = ReplHub::new(); + let snap_cfg = Arc::clone(&primary.state.snap); + let primary_store = Arc::clone(&primary.store); + let reg = Arc::clone(&repl_registry); + + // Replication listener — binds on port 0 + let repl_listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let repl_port = repl_listener.local_addr().unwrap().port(); + tokio::spawn(async move { + while let Ok((socket, _)) = repl_listener.accept().await { + let s = Arc::clone(&primary_store); + let sc = Arc::clone(&snap_cfg); + let r = Arc::clone(®); + tokio::spawn(handle_replica( + socket, + s, + sc, + r, + None, + DEFAULT_REPL_CHANNEL_CAPACITY, + IpAddr::from([127, 0, 0, 1]), + ReplAuthThrottle::new(), + )); + } + }); + + // Also wire the repl_registry into the primary state so on_write fans out + // We can't replace state.replicas (it's private), but handle_replica adds + // itself to the registry it receives. We pass the same repl_registry to + // on_write via a workaround: patch primary state's replicas after the fact + // by passing the same Arc. Since ServerState.replicas is private in our + // TestServer, we re-use the one we created. + // ── Simpler approach: replace on_write path by sharing registry ── + // Instead, wire it through the primary ServerState directly. + // (In practice the TestServer shares state.replicas which starts empty; + // handle_replica will push its sender into it when it connects.) + // The trick: we need primary.state.replicas to point to our repl_registry. + // Since TestServer.state is Arc, we can't replace it. + // Use a fresh primary state that shares our registry. + let primary2 = { + let store = Arc::clone(&primary.store); + let (tx, _rx) = broadcast::channel::(256); + let pubsub: SharedPubSub = Arc::new(tokio::sync::Mutex::new(PubSubHub::new())); + let wr: WatchRegistry = WatchHub::new(); + let sem = Arc::new(Semaphore::new(64)); + let snap = Arc::clone(&primary.state.snap); + let state = Arc::new(ServerState { + snap, + aof: None, + replicas: Arc::clone(&repl_registry), + is_replica: AtomicBool::new(false), + dedup: std::sync::Mutex::new(HashMap::new()), + ephemeral: std::sync::Mutex::new(HashMap::new()), + dedup_dirty: std::sync::atomic::AtomicBool::new(false), + }); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let store2 = Arc::clone(&store); + let state2 = Arc::clone(&state); + let pass = Arc::new(None::); + let task = tokio::spawn(async move { + loop { + let Ok((socket, _)) = listener.accept().await else { + return; + }; + let Ok(permit) = Arc::clone(&sem).try_acquire_owned() else { + continue; + }; + let (s, t, p, ps, wrr, st) = ( + Arc::clone(&store2), + tx.clone(), + Arc::clone(&pass), + Arc::clone(&pubsub), + Arc::clone(&wr), + Arc::clone(&state2), + ); + tokio::spawn(async move { + let peer = socket + .peer_addr() + .map(|a| a.to_string()) + .unwrap_or_default(); + handle_tcp(socket, s, t, p, ps, wrr, st, peer).await; + drop(permit); + }); + } + }); + TestServer { + tcp_addr: addr, + store, + state, + _task: task, + } + }; + + // Start replica + let replica_store = Arc::new(KeyValueStore::new()); + let replica_state = Arc::new(ServerState { + snap: Arc::new(SnapshotConfig { + path: tmp_path("repl_snap.rdb"), + last_save: AtomicI64::new(0), + }), + aof: None, + replicas: ReplHub::new(), + is_replica: AtomicBool::new(true), + dedup: std::sync::Mutex::new(HashMap::new()), + ephemeral: std::sync::Mutex::new(HashMap::new()), + dedup_dirty: std::sync::atomic::AtomicBool::new(false), + }); + let rs = Arc::clone(&replica_store); + let rst = Arc::clone(&replica_state); + let repl_addr = format!("127.0.0.1:{repl_port}"); + let rtx = broadcast::channel::(16).0; + tokio::spawn(async move { + run_repl_client(repl_addr, rs, rst, None, None, rtx, None).await; + }); + + // Give replica time to connect and receive initial snapshot + tokio::time::sleep(tokio::time::Duration::from_millis(300)).await; + + // Write to primary2 (which uses the shared repl_registry) + let mut c = RespClient::connect(primary2.tcp_addr).await; + assert_eq!(c.cmd(&["SET", "replkey", "replval"]).await, ok()); + + // Give replication fan-out time to arrive + tokio::time::sleep(tokio::time::Duration::from_millis(150)).await; + + assert_eq!( + replica_store.execute(Command::Get("replkey".into())), + Value::BulkString(Some(b"replval".to_vec())) + ); + + // The replica acknowledges what it applies, so once it has caught up the + // primary must observe zero lag. Before acknowledgements existed the + // primary had no way to distinguish this from a replica that had + // received the frame and silently failed to apply it. + let mut lag = u64::MAX; + for _ in 0..50 { + lag = repl_registry.max_lag_frames().await; + if lag == 0 { + break; + } + tokio::time::sleep(tokio::time::Duration::from_millis(20)).await; + } + assert_eq!(lag, 0, "caught-up replica must report zero lag"); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn integration_ws_accepts_binary_command_frames() { + // A WebSocket text frame must be well-formed UTF-8, so a command + // carrying raw bytes can only travel in a binary frame. The server + // previously handled text frames only and dropped binary ones. + let srv = spawn_ws_server().await; + let mut c = WsClient::connect(srv.tcp_addr).await; + + // A binary frame whose contents are valid UTF-8 is a normal command. + let raw = b"*3\r\n$3\r\nSET\r\n$3\r\nbin\r\n$5\r\nhello\r\n".to_vec(); + assert_eq!(c.cmd_binary(raw).await, ok()); + assert_eq!(c.cmd(&["GET", "bin"]).await, bulk("hello")); + + // A binary value round-trips byte-for-byte, and the reply comes back in + // a binary frame because it is not valid UTF-8. + let binary: &[u8] = &[0xff, 0xfe, 0x00, 0x41]; + let mut req = b"*3\r\n$3\r\nSET\r\n$3\r\nraw\r\n".to_vec(); + req.extend_from_slice(format!("${}\r\n", binary.len()).as_bytes()); + req.extend_from_slice(binary); + req.extend_from_slice(b"\r\n"); + assert_eq!(c.cmd_binary(req).await, ok()); + assert_eq!( + c.cmd(&["GET", "raw"]).await, + Value::BulkString(Some(binary.to_vec())) + ); + + // The connection stays usable. + assert_eq!(c.cmd(&["PING"]).await, Value::SimpleString("PONG".into())); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn integration_ws_hello_reports_resp3_and_refuses_downgrade() { + let srv = spawn_ws_server().await; + let mut c = WsClient::connect(srv.tcp_addr).await; + + // The sync protocol is defined in terms of RESP3 push frames. + let v = c.cmd(&["HELLO"]).await; + let Value::Map(pairs) = v else { + panic!("WS HELLO must reply with a RESP3 map, got {v:?}") + }; + assert!( + pairs + .iter() + .any(|(k, v)| *k == bulk("proto") && *v == Value::Integer(3)) + ); + + // Downgrading would silently break push delivery, so it is refused + // rather than accepted-and-ignored. + let v = c.cmd(&["HELLO", "2"]).await; + assert!( + matches!(&v, Value::Error(e) if e.starts_with("NOPROTO")), + "expected NOPROTO, got {v:?}" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn integration_ws_reaches_the_introspection_commands_too() { + // `handle_tcp` and `handle_ws` are two hand-maintained copies of one + // command loop, so the standing hazard when adding a server-level + // command is wiring it into one and not the other — which compiles, and + // fails only over the transport nobody checked. Every command added + // outside the store belongs in a test like this one. + let srv = spawn_ws_server().await; + let mut c = WsClient::connect(srv.tcp_addr).await; + + assert_eq!(c.cmd(&["SET", "k", "hello"]).await, ok()); + let usage = c.cmd(&["MEMORY", "USAGE", "k"]).await; + assert!( + matches!(usage, Value::Integer(n) if n > 0), + "MEMORY USAGE over WS: {usage:?}" + ); + assert_eq!( + c.cmd(&["MEMORY", "USAGE", "ghost"]).await, + Value::BulkString(None) + ); + assert!(matches!( + c.cmd(&["MEMORY", "DOCTOR"]).await, + Value::Error(_) + )); + + assert_eq!(c.cmd(&["MODULE", "LIST"]).await, Value::Array(Some(vec![]))); + assert!(matches!(c.cmd(&["CLUSTER", "INFO"]).await, Value::Error(_))); + + // No subscribers on this connection, so the registry is empty — the + // point is that the command is answered at all rather than falling + // through to the store's "handled by the connection layer" refusal. + assert_eq!( + c.cmd(&["PUBSUB", "CHANNELS"]).await, + Value::Array(Some(vec![])) + ); + assert_eq!(c.cmd(&["PUBSUB", "NUMPAT"]).await, Value::Integer(0)); +} + +#[tokio::test] +async fn replication_lag_counts_unacknowledged_frames() { + // A replica that receives frames but never acknowledges them is exactly + // the case queue depth cannot see: the frames left the primary's + // channel, so the queue reads empty while the replica is arbitrarily + // far behind. Lag must report them. + let store = Arc::new(KeyValueStore::new()); + let registry: ReplRegistry = ReplHub::new(); + let snap_cfg = Arc::new(SnapshotConfig { + path: tmp_path("lag_snap.rdb"), + last_save: AtomicI64::new(0), + }); + + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + { + let (s, sc, r) = ( + Arc::clone(&store), + Arc::clone(&snap_cfg), + Arc::clone(®istry), + ); + tokio::spawn(async move { + if let Ok((socket, _)) = listener.accept().await { + let _ = handle_replica( + socket, + s, + sc, + r, + None, + DEFAULT_REPL_CHANNEL_CAPACITY, + IpAddr::from([127, 0, 0, 1]), + ReplAuthThrottle::new(), + ) + .await; + } + }); + } + + // A replica that reads the snapshot and then goes silent. + let mut sock = TcpStream::connect(addr).await.unwrap(); + let mut len_buf = [0u8; 4]; + sock.read_exact(&mut len_buf).await.unwrap(); + let mut snap = vec![0u8; u32::from_le_bytes(len_buf) as usize]; + sock.read_exact(&mut snap).await.unwrap(); + + // Wait for registration, then fan out three writes. + for _ in 0..50 { + if registry.count.load(Ordering::Relaxed) == 1 { + break; + } + tokio::time::sleep(tokio::time::Duration::from_millis(20)).await; + } + for i in 0..3 { + registry + .fan_out(format!("*1\r\n$4\r\nPING{i}\r\n").into_bytes()) + .await; + } + + let mut lag = 0; + for _ in 0..50 { + lag = registry.max_lag_frames().await; + if lag == 3 { + break; + } + tokio::time::sleep(tokio::time::Duration::from_millis(20)).await; + } + assert_eq!(lag, 3, "three unacknowledged frames must show as lag 3"); + + // Acknowledging two of them retires exactly two frames of lag. + sock.write_all(&2u64.to_le_bytes()).await.unwrap(); + for _ in 0..50 { + lag = registry.max_lag_frames().await; + if lag == 1 { + break; + } + tokio::time::sleep(tokio::time::Duration::from_millis(20)).await; + } + assert_eq!(lag, 1, "after acking 2 of 3, one frame remains outstanding"); + + // A stale ack must not walk the high-water mark backwards. + sock.write_all(&1u64.to_le_bytes()).await.unwrap(); + tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + assert_eq!( + registry.max_lag_frames().await, + 1, + "a replayed lower ack must not increase reported lag" + ); +} + +// ── Integration: 3e load (ignored in normal CI) ─────────────────────────── + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore] +async fn integration_concurrent_writers() { + let srv = Arc::new(spawn_server().await); + let addr = srv.tcp_addr; + + let tasks: Vec<_> = (0..50) + .map(|task_id| { + tokio::spawn(async move { + let mut c = RespClient::connect(addr).await; + for i in 0..100u32 { + let key = format!("t{task_id}_{i}"); + let val = format!("v{i}"); + assert_eq!(c.cmd(&["SET", &key, &val]).await, ok()); + assert_eq!(c.cmd(&["GET", &key]).await, bulk(&val)); + } + }) + }) + .collect(); + + for t in tasks { + t.await.unwrap(); + } + // All 50 × 100 keys should be present + assert_eq!(srv.store.execute(Command::DbSize), Value::Integer(5000)); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore] +async fn integration_connection_limit() { + // Small semaphore: only 3 concurrent connections + let store = Arc::new(KeyValueStore::new()); + let (tx, _rx) = broadcast::channel::(16); + let pubsub: SharedPubSub = Arc::new(tokio::sync::Mutex::new(PubSubHub::new())); + let watch_registry: WatchRegistry = WatchHub::new(); + let semaphore = Arc::new(Semaphore::new(3)); + let state = Arc::new(ServerState { + snap: Arc::new(SnapshotConfig { + path: tmp_path("conn_limit.rdb"), + last_save: AtomicI64::new(0), + }), + aof: None, + replicas: ReplHub::new(), + is_replica: AtomicBool::new(false), + dedup: std::sync::Mutex::new(HashMap::new()), + ephemeral: std::sync::Mutex::new(HashMap::new()), + dedup_dirty: std::sync::atomic::AtomicBool::new(false), + }); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let store2 = Arc::clone(&store); + let state2 = Arc::clone(&state); + let pass = Arc::new(None::); + + tokio::spawn(async move { + loop { + let Ok((socket, _)) = listener.accept().await else { + return; + }; + let Ok(permit) = Arc::clone(&semaphore).try_acquire_owned() else { + // Drop socket immediately — connection limit reached + drop(socket); + continue; + }; + let (s, t, p, ps, wr, st) = ( + Arc::clone(&store2), + tx.clone(), + Arc::clone(&pass), + Arc::clone(&pubsub), + Arc::clone(&watch_registry), + Arc::clone(&state2), + ); + tokio::spawn(async move { + let peer = socket + .peer_addr() + .map(|a| a.to_string()) + .unwrap_or_default(); + handle_tcp(socket, s, t, p, ps, wr, st, peer).await; + drop(permit); + }); + } + }); + + // Open 3 connections and hold them (just send PING and keep the socket open) + let mut holders = Vec::new(); + for _ in 0..3 { + let mut c = RespClient::connect(addr).await; + assert_eq!( + c.cmd(&["PING"]).await, + Value::SimpleString("PONG".to_string()) + ); + holders.push(c); + } + + // 4th connection: server drops it immediately, so read returns 0 + let mut overflow = TcpStream::connect(addr).await.unwrap(); + let mut buf = [0u8; 64]; + let n = overflow.read(&mut buf).await.unwrap_or(0); + assert_eq!(n, 0, "4th connection should have been closed by server"); + + drop(holders); +} + +// ── Integration: 3f chaos (ignored in normal CI) ────────────────────────── + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +#[ignore] +async fn integration_kill_primary_mid_write() { + let srv = Arc::new(spawn_server().await); + let addr = srv.tcp_addr; + + // Start 20 concurrent writers + let tasks: Vec<_> = (0..20) + .map(|i| { + tokio::spawn(async move { + // Connect; tolerate connection errors (server may die mid-flight) + let stream = TcpStream::connect(addr).await; + if stream.is_err() { + return; + } + let mut c = RespClient { + stream: stream.unwrap(), + buf: vec![0u8; 65536], + filled: 0, + }; + for j in 0..50u32 { + let key = format!("chaos_{i}_{j}"); + // Ignore errors — server may die during this + let _ = tokio::time::timeout( + tokio::time::Duration::from_millis(200), + c.cmd(&["SET", &key, "v"]), + ) + .await; + } + }) + }) + .collect(); + + // Kill the server after 10ms + tokio::time::sleep(tokio::time::Duration::from_millis(10)).await; + srv._task.abort(); + + // Join all writers — none should panic + for t in tasks { + let _ = t.await; + } + + // Store is still intact in memory — no panic is the meaningful assertion here; + // zero keys is valid if the server was killed before any write landed. +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[ignore] +async fn integration_failover_promotes() { + // Point replica at a port that refuses connections immediately so the + // unreachable timer starts on the first loop iteration without any + // real primary required. Promotion happens after: + // connect fail (fast) → backoff 2s → connect fail → elapsed ≥ 1s → promote + // so we wait 3s to be safe. + let replica_state = Arc::new(ServerState { + snap: Arc::new(SnapshotConfig { + path: tmp_path("failover_snap.rdb"), + last_save: AtomicI64::new(0), + }), + aof: None, + replicas: ReplHub::new(), + is_replica: AtomicBool::new(true), + dedup: std::sync::Mutex::new(HashMap::new()), + ephemeral: std::sync::Mutex::new(HashMap::new()), + dedup_dirty: std::sync::atomic::AtomicBool::new(false), + }); + let replica_store = Arc::new(KeyValueStore::new()); + let rs = Arc::clone(&replica_store); + let rst = Arc::clone(&replica_state); + // Bind a listener then immediately drop it so the port is known-refused + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let dead_addr = format!("127.0.0.1:{}", listener.local_addr().unwrap().port()); + drop(listener); + let rtx = broadcast::channel::(16).0; + tokio::spawn(async move { + run_repl_client(dead_addr, rs, rst, None, Some(1), rtx, None).await; + }); + + // Wait for 2 backoff cycles (initial fail + 2s sleep + retry fail → promote) + tokio::time::sleep(tokio::time::Duration::from_millis(3000)).await; + + assert!( + !replica_state.is_replica(), + "replica should have promoted after primary was unreachable for >1s" + ); +} + +// ── WebSocket WATCH/EXEC harness ────────────────────────────────────────── + +/// Spawn a WebSocket server sharing one store + watch registry across all +/// connections, so WATCH notifications fan out between clients. +async fn spawn_ws_server() -> TestServer { + spawn_ws_server_cfg(None).await +} + +/// Like `spawn_ws_server`, with an optional sync-scope secret (strict mode). +async fn spawn_ws_server_cfg(sync_secret: Option) -> TestServer { + spawn_ws_server_full(sync_secret, None).await +} + +/// Like `spawn_ws_server`, with an origin allowlist in force. +async fn spawn_ws_server_origins(origins: Vec) -> TestServer { + spawn_ws_server_full(None, Some(origins)).await +} + +/// Open a WebSocket to `addr`, optionally sending an `Origin` header, and +/// report whether the handshake completed. +async fn ws_connect_with_origin( + addr: std::net::SocketAddr, + origin: Option<&str>, +) -> Result<(), String> { + use tokio_tungstenite::tungstenite::client::IntoClientRequest; + let mut req = format!("ws://{addr}").into_client_request().unwrap(); + if let Some(o) = origin { + req.headers_mut().insert("origin", o.parse().unwrap()); + } + tokio_tungstenite::connect_async(req) + .await + .map(|_| ()) + .map_err(|e| e.to_string()) +} + +#[tokio::test] +async fn ws_refuses_a_cross_origin_handshake() { + // Browsers apply neither CORS nor a preflight to WebSockets, so before + // this check any page a user visited could open a socket to port 6380 + // and read or write the whole keyspace with that user's network + // position. On ws://localhost:6380 that is every site in every tab. + let srv = spawn_ws_server_origins(vec!["https://app.example.com".to_string()]).await; + + let err = ws_connect_with_origin(srv.tcp_addr, Some("https://evil.example")) + .await + .expect_err("a foreign Origin must not complete the handshake"); + assert!( + err.contains("403") || err.to_lowercase().contains("forbidden"), + "expected a 403, got {err}" + ); +} + +#[tokio::test] +async fn ws_admits_an_allowlisted_origin_and_serves_commands() { + // The rejection is worthless if it also breaks the deployed app, so + // assert the permitted path all the way through to a working command. + let srv = spawn_ws_server_origins(vec!["https://app.example.com".to_string()]).await; + ws_connect_with_origin(srv.tcp_addr, Some("https://app.example.com")) + .await + .expect("an allowlisted Origin must connect"); + + let mut c = WsClient::connect(srv.tcp_addr).await; + assert_eq!(c.cmd(&["SET", "k", "v"]).await, ok()); + assert_eq!(c.cmd(&["GET", "k"]).await, bulk("v")); +} + +#[tokio::test] +async fn ws_admits_a_client_that_sends_no_origin() { + // Native clients omit the header, and an attacker with a raw socket can + // forge it — refusing here would break real clients while stopping + // nobody. `WsClient::connect` is exactly such a client. + let srv = spawn_ws_server_origins(vec!["https://app.example.com".to_string()]).await; + ws_connect_with_origin(srv.tcp_addr, None) + .await + .expect("a client with no Origin must connect"); +} + +#[tokio::test] +async fn ws_without_an_allowlist_accepts_any_origin() { + // Unset means allow, matching RECACHED_PASSWORD. The startup warning is + // what keeps this from being a silent default. + let srv = spawn_ws_server().await; + ws_connect_with_origin(srv.tcp_addr, Some("https://anything.example")) + .await + .expect("no allowlist means no origin restriction"); +} + +async fn spawn_ws_server_full( + sync_secret: Option, + allowed_origins: Option>, +) -> TestServer { + let store = Arc::new(KeyValueStore::new()); + let (tx, _rx) = broadcast::channel::(256); + let pubsub: SharedPubSub = Arc::new(tokio::sync::Mutex::new(PubSubHub::new())); + let watch_registry: WatchRegistry = WatchHub::new(); + let snap_cfg = Arc::new(SnapshotConfig { + path: tmp_path("ws_test.rdb"), + last_save: AtomicI64::new(now_unix_secs()), + }); + let state = Arc::new(ServerState { + snap: snap_cfg, + aof: None, + replicas: ReplHub::new(), + is_replica: AtomicBool::new(false), + dedup: std::sync::Mutex::new(HashMap::new()), + ephemeral: std::sync::Mutex::new(HashMap::new()), + dedup_dirty: std::sync::atomic::AtomicBool::new(false), + }); + + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let store2 = Arc::clone(&store); + let state2 = Arc::clone(&state); + let secret = Arc::new(sync_secret); + let origins = Arc::new(allowed_origins); + + let task = tokio::spawn(async move { + loop { + let Ok((socket, _)) = listener.accept().await else { + return; + }; + let (s, t, ps, wr, st, ss, ao) = ( + Arc::clone(&store2), + tx.clone(), + Arc::clone(&pubsub), + Arc::clone(&watch_registry), + Arc::clone(&state2), + Arc::clone(&secret), + Arc::clone(&origins), + ); + let id = next_conn_id(); + let peer = socket + .peer_addr() + .map(|a| a.to_string()) + .unwrap_or_default(); + tokio::spawn(async move { + handle_ws(socket, s, t, Arc::new(None), id, ps, wr, st, ss, ao, peer).await; + }); + } + }); + + TestServer { + tcp_addr: addr, + store, + state, + _task: task, + } +} + +struct WsClient { + ws: tokio_tungstenite::WebSocketStream>, +} + +impl WsClient { + async fn connect(addr: std::net::SocketAddr) -> Self { + let (ws, _) = tokio_tungstenite::connect_async(format!("ws://{addr}")) + .await + .unwrap(); + Self { ws } + } + + async fn cmd(&mut self, args: &[&str]) -> Value { + let mut req = format!("*{}\r\n", args.len()); + for a in args { + req.push_str(&format!("${}\r\n{}\r\n", a.len(), a)); + } + self.ws.send(Message::Text(req.into())).await.unwrap(); + self.next_reply().await + } + + /// Send pre-encoded RESP bytes in a *binary* frame. Text frames must be + /// well-formed UTF-8 per the WebSocket spec, so this is the only way to + /// put arbitrary bytes on the wire. + async fn cmd_binary(&mut self, raw: Vec) -> Value { + self.ws.send(Message::Binary(raw.into())).await.unwrap(); + self.next_reply().await + } + + /// Wait up to `ms` for the next frame of any kind — RESP3 Push + /// broadcasts *and* plain arrays. `keychange` notifications are encoded + /// as arrays, so `recv_push` skips them entirely. + async fn recv_any(&mut self, ms: u64) -> Option { + let fut = async { + loop { + match self.ws.next().await { + Some(Ok(Message::Text(t))) => return Some(t.to_string()), + Some(Ok(_)) => continue, + _ => return None, + } + } + }; + tokio::time::timeout(tokio::time::Duration::from_millis(ms), fut) + .await + .ok() + .flatten() + } + + /// Wait up to `ms` for the next RESP3 Push broadcast frame, returning + /// its raw text. `None` when nothing arrives in time. + async fn recv_push(&mut self, ms: u64) -> Option { + let fut = async { + loop { + match self.ws.next().await { + Some(Ok(Message::Text(t))) => { + let Ok((v, _)) = Value::parse(t.as_bytes()) else { + continue; + }; + if matches!(v, Value::Push(_)) { + return Some(t.to_string()); + } + } + Some(Ok(_)) => continue, + _ => return None, + } + } + }; + tokio::time::timeout(tokio::time::Duration::from_millis(ms), fut) + .await + .ok() + .flatten() + } + + /// Wait up to `ms` for the next `keychange` frame (WATCH / live-query + /// push), returning `(key, value)`. `None` when nothing arrives. + async fn recv_keychange(&mut self, ms: u64) -> Option<(String, Value)> { + let fut = async { + loop { + match self.ws.next().await { + Some(Ok(Message::Text(t))) => { + let Ok((v, _)) = Value::parse(t.as_bytes()) else { + continue; + }; + if let Value::Array(Some(items)) = &v + && items.len() == 3 + && matches!(items.first(), Some(Value::BulkString(Some(k))) if k == b"keychange") + { + let Value::BulkString(Some(key)) = &items[1] else { + continue; + }; + return Some(( + String::from_utf8_lossy(key).into_owned(), + items[2].clone(), + )); + } + } + Some(Ok(_)) => continue, + _ => return None, + } + } + }; + tokio::time::timeout(tokio::time::Duration::from_millis(ms), fut) + .await + .ok() + .flatten() + } + + /// Read the next *command reply*, skipping server-initiated frames + /// (RESP3 Push broadcasts and `keychange` observable-key pushes). + async fn next_reply(&mut self) -> Value { + loop { + let raw: Vec = match self.ws.next().await { + Some(Ok(Message::Text(t))) => t.as_bytes().to_vec(), + Some(Ok(Message::Binary(b))) => b.to_vec(), + Some(Ok(_)) => continue, + _ => panic!("ws closed unexpectedly"), + }; + let Ok((v, _)) = Value::parse(&raw) else { + continue; + }; + if matches!(v, Value::Push(_)) { + continue; + } + if let Value::Array(Some(items)) = &v + && matches!(items.first(), Some(Value::BulkString(Some(k))) if k == b"keychange") + { + continue; + } + return v; + } + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn integration_ws_watch_exec_aborts_on_change() { + let srv = spawn_ws_server().await; + let mut watcher = WsClient::connect(srv.tcp_addr).await; + let mut writer = WsClient::connect(srv.tcp_addr).await; + + assert_eq!(watcher.cmd(&["SET", "k", "v0"]).await, ok()); + assert_eq!(watcher.cmd(&["WATCH", "k"]).await, ok()); + + // Another client mutates the watched key. + assert_eq!(writer.cmd(&["SET", "k", "v1"]).await, ok()); + // Give the notification time to reach the watcher's registry channel. + tokio::time::sleep(tokio::time::Duration::from_millis(50)).await; + + assert_eq!( + watcher.cmd(&["MULTI"]).await, + Value::SimpleString("OK".into()) + ); + assert_eq!( + watcher.cmd(&["SET", "k", "v2"]).await, + Value::SimpleString("QUEUED".into()) + ); + // EXEC must abort with a nil array because k changed since WATCH. + assert_eq!(watcher.cmd(&["EXEC"]).await, Value::Array(None)); + // The transaction did not run. + assert_eq!(srv.store.execute(Command::Get("k".into())), bulk("v1")); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn integration_ws_watch_exec_runs_when_unchanged() { + let srv = spawn_ws_server().await; + let mut c = WsClient::connect(srv.tcp_addr).await; + + assert_eq!(c.cmd(&["WATCH", "k"]).await, ok()); + assert_eq!(c.cmd(&["MULTI"]).await, ok()); + assert_eq!( + c.cmd(&["SET", "k", "v1"]).await, + Value::SimpleString("QUEUED".into()) + ); + // No one touched k → EXEC runs and returns the queued results. + assert_eq!( + c.cmd(&["EXEC"]).await, + Value::Array(Some(vec![Value::SimpleString("OK".into())])) + ); + assert_eq!(srv.store.execute(Command::Get("k".into())), bulk("v1")); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn integration_tcp_watch_exec_aborts_on_change() { + let srv = spawn_server().await; + let mut watcher = RespClient::connect(srv.tcp_addr).await; + let mut writer = RespClient::connect(srv.tcp_addr).await; + + assert_eq!(watcher.cmd(&["SET", "k", "v0"]).await, ok()); + assert_eq!(watcher.cmd(&["WATCH", "k"]).await, ok()); + // Another client mutates the watched key (reply awaited → notification queued). + assert_eq!(writer.cmd(&["SET", "k", "v1"]).await, ok()); + + assert_eq!(watcher.cmd(&["MULTI"]).await, ok()); + assert_eq!( + watcher.cmd(&["SET", "k", "v2"]).await, + Value::SimpleString("QUEUED".into()) + ); + // k changed since WATCH → EXEC aborts with a nil array. + assert_eq!(watcher.cmd(&["EXEC"]).await, Value::Array(None)); + assert_eq!(watcher.cmd(&["GET", "k"]).await, bulk("v1")); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn integration_tcp_watch_exec_runs_when_unchanged() { + let srv = spawn_server().await; + let mut c = RespClient::connect(srv.tcp_addr).await; + + assert_eq!(c.cmd(&["WATCH", "k"]).await, ok()); + assert_eq!(c.cmd(&["MULTI"]).await, ok()); + assert_eq!( + c.cmd(&["SET", "k", "v1"]).await, + Value::SimpleString("QUEUED".into()) + ); + assert_eq!( + c.cmd(&["EXEC"]).await, + Value::Array(Some(vec![Value::SimpleString("OK".into())])) + ); + assert_eq!(c.cmd(&["GET", "k"]).await, bulk("v1")); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn integration_tcp_watch_inside_multi_rejected() { + let srv = spawn_server().await; + let mut c = RespClient::connect(srv.tcp_addr).await; + assert_eq!(c.cmd(&["MULTI"]).await, ok()); + // WATCH is not allowed once a transaction has started. + assert!(matches!(c.cmd(&["WATCH", "k"]).await, Value::Error(_))); +} + +// ── Sync scoping ────────────────────────────────────────────────────────── + +/// Mint a sync-scope token the way an application backend would: +/// HMAC-SHA256 over the base64url payload text. +fn mint_sync_token(secret: &str, payload: &str) -> String { + use base64::Engine as _; + use hmac::{Hmac, Mac}; + let engine = base64::engine::general_purpose::URL_SAFE_NO_PAD; + let payload_b64 = engine.encode(payload); + let mut mac = Hmac::::new_from_slice(secret.as_bytes()).unwrap(); + mac.update(payload_b64.as_bytes()); + let sig = engine.encode(mac.finalize().into_bytes()); + format!("{payload_b64}.{sig}") +} + +#[test] +fn sync_token_roundtrip_and_rejections() { + let tok = mint_sync_token("s3cret", "cart:42:*,profile:42"); + assert_eq!( + verify_sync_token("s3cret", &tok).unwrap(), + vec!["cart:42:*".to_string(), "profile:42".to_string()] + ); + // Wrong secret → invalid signature. + assert_eq!( + verify_sync_token("other", &tok).unwrap_err(), + "invalid signature" + ); + // Tampered payload → invalid signature. + let (_, sig) = tok.split_once('.').unwrap(); + use base64::Engine as _; + let engine = base64::engine::general_purpose::URL_SAFE_NO_PAD; + let forged = format!("{}.{}", engine.encode("admin:*"), sig); + assert_eq!( + verify_sync_token("s3cret", &forged).unwrap_err(), + "invalid signature" + ); + // Expired token. + let expired = mint_sync_token("s3cret", "cart:*|1"); + assert_eq!( + verify_sync_token("s3cret", &expired).unwrap_err(), + "token expired" + ); + // Future expiry still valid. + let live = mint_sync_token("s3cret", "cart:*|99999999999"); + assert!(verify_sync_token("s3cret", &live).is_ok()); + // Empty patterns / malformed. + let empty = mint_sync_token("s3cret", ""); + assert_eq!( + verify_sync_token("s3cret", &empty).unwrap_err(), + "token grants no patterns" + ); + assert!(verify_sync_token("s3cret", "no-dot-here").is_err()); +} + +#[test] +fn scopes_match_globs_and_flushdb() { + let scopes = vec!["cart:42:*".to_string(), "catalog:*".to_string()]; + assert!(scopes_match(&scopes, &["cart:42:item:1".to_string()])); + assert!(scopes_match(&scopes, &["catalog:books".to_string()])); + assert!(!scopes_match(&scopes, &["cart:7:item:1".to_string()])); + assert!(!scopes_match(&scopes, &["session:42".to_string()])); + // Multi-key: any matching key makes the push visible. + assert!(scopes_match( + &scopes, + &["session:42".to_string(), "catalog:books".to_string()] + )); + // No keys = FLUSHDB — visible to every scope. + assert!(scopes_match(&scopes, &[])); +} + +#[test] +fn command_scope_classification() { + assert!(matches!( + command_scope(&Command::Ping(None)), + CommandScope::KeyLess + )); + assert!(matches!( + command_scope(&Command::Keys("*".into())), + CommandScope::Admin + )); + assert!(matches!( + command_scope(&Command::FlushDb), + CommandScope::Admin + )); + match command_scope(&Command::Get("a".into())) { + CommandScope::Keys(k) => assert_eq!(k, vec!["a".to_string()]), + _ => panic!("GET should be key-scoped"), + } + match command_scope(&Command::SInterStore( + "dst".into(), + vec!["a".into(), "b".into()], + )) { + CommandScope::Keys(k) => { + assert!(k.contains(&"dst".to_string()) && k.contains(&"a".to_string())) + } + _ => panic!("SINTERSTORE should be key-scoped"), + } +} + +// ── Scope enforcement: the authorization surface ────────────────────────── +// +// `command_scope` decides whether a command is checked against the +// connection's grants at all. A key-touching command misclassified as +// `KeyLess` skips the check entirely, so every family is pinned here. +// The match in `command_scope` is exhaustive over `Command` with no +// catch-all — a new variant fails to compile until classified — and these +// tests guard against the remaining risk: classifying one wrongly. + +fn scoped_keys(cmd: Command) -> Vec { + match command_scope(&cmd) { + CommandScope::Keys(k) => k, + other => panic!("{cmd:?} should be key-scoped, got {other:?}"), + } +} + +#[test] +fn every_single_key_command_family_is_key_scoped() { + let cases: Vec<(Command, &str)> = vec![ + (Command::Get("k".into()), "k"), + ( + Command::Set("k".into(), "v".into(), SetOptions::default()), + "k", + ), + (Command::Append("k".into(), "v".into()), "k"), + (Command::Incr("k".into()), "k"), + (Command::Decr("k".into()), "k"), + (Command::Ttl("k".into()), "k"), + (Command::Persist("k".into()), "k"), + (Command::Expire("k".into(), 1), "k"), + (Command::Type("k".into()), "k"), + (Command::HGet("k".into(), "f".into()), "k"), + (Command::HGetAll("k".into()), "k"), + (Command::LPush("k".into(), vec!["v".into()]), "k"), + (Command::LRange("k".into(), 0, -1), "k"), + (Command::SAdd("k".into(), vec!["m".into()]), "k"), + (Command::SMembers("k".into()), "k"), + ( + Command::ZAdd("k".into(), ZAddOptions::default(), vec![(1.0, "m".into())]), + "k", + ), + (Command::ZScore("k".into(), "m".into()), "k"), + (Command::JGet("k".into(), None), "k"), + (Command::JSet("k".into(), "$".into(), "1".into()), "k"), + (Command::RlCheck("k".into(), None), "k"), + ]; + for (cmd, expected) in cases { + let keys = scoped_keys(cmd.clone()); + assert!( + keys.contains(&expected.to_string()), + "{cmd:?} must report key '{expected}', got {keys:?}" + ); + } +} + +#[test] +fn multi_key_commands_report_every_key_they_touch() { + // A key omitted here is a key that never gets scope-checked, because + // `scopes_match` only inspects the keys it is handed. + let rename = scoped_keys(Command::Rename("src".into(), "dst".into())); + assert!( + rename.contains(&"src".to_string()) && rename.contains(&"dst".to_string()), + "RENAME touches both keys, got {rename:?}" + ); + + let smove = scoped_keys(Command::SMove("src".into(), "dst".into(), "m".into())); + assert!( + smove.contains(&"src".to_string()) && smove.contains(&"dst".to_string()), + "SMOVE touches both sets, got {smove:?}" + ); + + let mset = scoped_keys(Command::MSet(vec![ + ("a".into(), "1".into()), + ("b".into(), "2".into()), + ])); + assert!( + mset.contains(&"a".to_string()) && mset.contains(&"b".to_string()), + "MSET touches every key, got {mset:?}" + ); + + let mget = scoped_keys(Command::MGet(vec!["a".into(), "b".into()])); + assert!(mget.contains(&"a".to_string()) && mget.contains(&"b".to_string())); + + let del = scoped_keys(Command::Del(vec!["a".into(), "b".into()])); + assert!(del.contains(&"a".to_string()) && del.contains(&"b".to_string())); + + let exists = scoped_keys(Command::Exists(vec!["a".into(), "b".into()])); + assert!(exists.contains(&"a".to_string()) && exists.contains(&"b".to_string())); + + let sunion = scoped_keys(Command::SUnionStore( + "dst".into(), + vec!["a".into(), "b".into()], + )); + for k in ["dst", "a", "b"] { + assert!(sunion.contains(&k.to_string()), "SUNIONSTORE missed {k}"); + } +} + +#[test] +fn administrative_commands_are_denied_not_merely_unscoped() { + // These would read or destroy data outside any grant, so they must map + // to Admin (refused) rather than KeyLess (silently allowed). + for cmd in [ + Command::Keys("*".into()), + Command::Scan(0, None, None), + Command::DbSize, + Command::FlushDb, + Command::Save, + Command::BgSave, + Command::LastSave, + Command::ReplicaOfNoOne, + ] { + assert!( + matches!(command_scope(&cmd), CommandScope::Admin), + "{cmd:?} must be Admin-classified" + ); + } +} + +#[test] +fn keyless_commands_touch_no_keys() { + for cmd in [ + Command::Ping(None), + Command::Auth("pw".into()), + Command::Multi, + Command::Exec, + Command::Discard, + Command::Subscribe(vec!["ch".into()]), + Command::Publish("ch".into(), "m".into()), + Command::Sync(vec![]), + ] { + assert!( + matches!(command_scope(&cmd), CommandScope::KeyLess), + "{cmd:?} should be KeyLess" + ); + } +} + +#[test] +fn dedup_envelope_inherits_the_inner_command_scope() { + // DEDUP wraps a real write. If the wrapper were treated as KeyLess, an + // attacker could smuggle any command past scope enforcement. + let inner = Command::Set("secret:1".into(), "v".into(), SetOptions::default()); + let wrapped = Command::Dedup("client".into(), 1, Box::new(inner)); + match command_scope(&wrapped) { + CommandScope::Keys(k) => assert_eq!(k, vec!["secret:1".to_string()]), + other => panic!("DEDUP must inherit inner scope, got {other:?}"), + } + + // The same must hold for an admin inner command. + let wrapped_admin = Command::Dedup("client".into(), 2, Box::new(Command::FlushDb)); + assert!(matches!(command_scope(&wrapped_admin), CommandScope::Admin)); +} + +#[test] +fn scopes_match_allows_when_there_are_no_keys_to_check() { + // Documented consequence of the design: an empty key list is allowed. + // That is only safe because `command_scope` returns `Keys(..)` for every + // key-touching command — the test above is what keeps it safe. + assert!(scopes_match(&["cart:*".to_string()], &[])); +} + +#[test] +fn scopes_match_requires_every_key_to_match() { + let scopes = vec!["cart:42:*".to_string()]; + assert!(scopes_match(&scopes, &["cart:42:a".to_string()])); + // One in-scope key does not license an out-of-scope sibling. + assert!(!scopes_match(&scopes, &["cart:99:a".to_string()])); +} + +// ── Metrics labels ──────────────────────────────────────────────────────── + +// ── Exhaustive command classification ───────────────────────────────────── +// +// One instance of every `Command` variant, run through the three functions +// that decide scope enforcement and metrics. `command_scope` has no +// catch-all arm, so a new variant cannot compile without being classified — +// but nothing stops it being classified *wrongly*, and a key-touching +// command marked `KeyLess` silently bypasses scope checks entirely. + +enum Expect { + KeyLess, + Admin, + Keys(&'static [&'static str]), +} + +fn all_commands() -> Vec<(Command, Expect)> { + vec![ + (Command::Ping(None), Expect::KeyLess), + (Command::Auth("pw".into()), Expect::KeyLess), + ( + Command::Set("k".into(), "v".into(), SetOptions::default()), + Expect::Keys(&["k"]), + ), + (Command::Get("k".into()), Expect::Keys(&["k"])), + (Command::ESet("k".into(), "v".into()), Expect::Keys(&["k"])), + (Command::Del(vec!["k".into()]), Expect::Keys(&["k"])), + (Command::Unlink(vec!["k".into()]), Expect::Keys(&["k"])), + ( + Command::Append("k".into(), "v".into()), + Expect::Keys(&["k"]), + ), + (Command::Strlen("k".into()), Expect::Keys(&["k"])), + (Command::GetRange("k".into(), 0, -1), Expect::Keys(&["k"])), + ( + Command::GetSet("k".into(), "v".into()), + Expect::Keys(&["k"]), + ), + (Command::MGet(vec!["k".into()]), Expect::Keys(&["k"])), + (Command::SetNx("k".into(), "v".into()), Expect::Keys(&["k"])), + ( + Command::SetEx("k".into(), 1, "v".into()), + Expect::Keys(&["k"]), + ), + ( + Command::PSetEx("k".into(), 1, "v".into()), + Expect::Keys(&["k"]), + ), + ( + Command::MSet(vec![("k".into(), "v".into())]), + Expect::Keys(&["k"]), + ), + (Command::Incr("k".into()), Expect::Keys(&["k"])), + (Command::Decr("k".into()), Expect::Keys(&["k"])), + (Command::IncrBy("k".into(), 1), Expect::Keys(&["k"])), + (Command::DecrBy("k".into(), 1), Expect::Keys(&["k"])), + (Command::Expire("k".into(), 1), Expect::Keys(&["k"])), + (Command::PExpire("k".into(), 1), Expect::Keys(&["k"])), + (Command::ExpireAt("k".into(), 1), Expect::Keys(&["k"])), + (Command::PExpireAt("k".into(), 1), Expect::Keys(&["k"])), + (Command::Ttl("k".into()), Expect::Keys(&["k"])), + (Command::PTtl("k".into()), Expect::Keys(&["k"])), + (Command::Persist("k".into()), Expect::Keys(&["k"])), + (Command::Exists(vec!["k".into()]), Expect::Keys(&["k"])), + (Command::Keys("*".into()), Expect::Admin), + (Command::Scan(0, None, None), Expect::Admin), + (Command::DbSize, Expect::Admin), + (Command::FlushDb, Expect::Admin), + ( + Command::Rename("k".into(), "d".into()), + Expect::Keys(&["k", "d"]), + ), + (Command::Type("k".into()), Expect::Keys(&["k"])), + ( + Command::HSet("k".into(), vec![("f".into(), "v".into())]), + Expect::Keys(&["k"]), + ), + (Command::HGet("k".into(), "f".into()), Expect::Keys(&["k"])), + (Command::HGetAll("k".into()), Expect::Keys(&["k"])), + ( + Command::HDel("k".into(), vec!["f".into()]), + Expect::Keys(&["k"]), + ), + (Command::HKeys("k".into()), Expect::Keys(&["k"])), + (Command::HVals("k".into()), Expect::Keys(&["k"])), + (Command::HLen("k".into()), Expect::Keys(&["k"])), + ( + Command::HIncrBy("k".into(), "f".into(), 1), + Expect::Keys(&["k"]), + ), + ( + Command::HIncrByFloat("k".into(), "f".into(), 1.0), + Expect::Keys(&["k"]), + ), + ( + Command::HExists("k".into(), "f".into()), + Expect::Keys(&["k"]), + ), + ( + Command::HSetNx("k".into(), "f".into(), "v".into()), + Expect::Keys(&["k"]), + ), + ( + Command::HMGet("k".into(), vec!["f".into()]), + Expect::Keys(&["k"]), + ), + ( + Command::HScan("k".into(), ScanArgs::default()), + Expect::Keys(&["k"]), + ), + ( + Command::LPush("k".into(), vec!["v".into()]), + Expect::Keys(&["k"]), + ), + ( + Command::RPush("k".into(), vec!["v".into()]), + Expect::Keys(&["k"]), + ), + ( + Command::LPushX("k".into(), vec!["v".into()]), + Expect::Keys(&["k"]), + ), + ( + Command::RPushX("k".into(), vec!["v".into()]), + Expect::Keys(&["k"]), + ), + (Command::LPop("k".into(), None), Expect::Keys(&["k"])), + (Command::RPop("k".into(), None), Expect::Keys(&["k"])), + (Command::LRange("k".into(), 0, -1), Expect::Keys(&["k"])), + (Command::LLen("k".into()), Expect::Keys(&["k"])), + (Command::LIndex("k".into(), 0), Expect::Keys(&["k"])), + ( + Command::LSet("k".into(), 0, "v".into()), + Expect::Keys(&["k"]), + ), + ( + Command::LRem("k".into(), 0, "v".into()), + Expect::Keys(&["k"]), + ), + (Command::LTrim("k".into(), 0, -1), Expect::Keys(&["k"])), + ( + Command::SAdd("k".into(), vec!["m".into()]), + Expect::Keys(&["k"]), + ), + (Command::SMembers("k".into()), Expect::Keys(&["k"])), + ( + Command::SRem("k".into(), vec!["m".into()]), + Expect::Keys(&["k"]), + ), + (Command::SCard("k".into()), Expect::Keys(&["k"])), + ( + Command::SIsMember("k".into(), "m".into()), + Expect::Keys(&["k"]), + ), + ( + Command::SMIsMember("k".into(), vec!["m".into()]), + Expect::Keys(&["k"]), + ), + (Command::SInter(vec!["k".into()]), Expect::Keys(&["k"])), + ( + Command::SInterStore("d".into(), vec!["k".into()]), + Expect::Keys(&["k", "d"]), + ), + (Command::SUnion(vec!["k".into()]), Expect::Keys(&["k"])), + ( + Command::SUnionStore("d".into(), vec!["k".into()]), + Expect::Keys(&["k", "d"]), + ), + (Command::SDiff(vec!["k".into()]), Expect::Keys(&["k"])), + ( + Command::SDiffStore("d".into(), vec!["k".into()]), + Expect::Keys(&["k", "d"]), + ), + (Command::SPop("k".into(), None), Expect::Keys(&["k"])), + (Command::SRandMember("k".into(), None), Expect::Keys(&["k"])), + ( + Command::SScan("k".into(), ScanArgs::default()), + Expect::Keys(&["k"]), + ), + ( + Command::SMove("k".into(), "d".into(), "m".into()), + Expect::Keys(&["k", "d"]), + ), + ( + Command::ZAdd("k".into(), ZAddOptions::default(), vec![(1.0, "m".into())]), + Expect::Keys(&["k"]), + ), + ( + Command::ZRange("k".into(), 0, -1, false), + Expect::Keys(&["k"]), + ), + ( + Command::ZRevRange("k".into(), 0, -1, false), + Expect::Keys(&["k"]), + ), + ( + Command::ZRangeByScore("k".into(), "0".into(), "1".into(), false, None), + Expect::Keys(&["k"]), + ), + ( + Command::ZRevRangeByScore("k".into(), "1".into(), "0".into(), false, None), + Expect::Keys(&["k"]), + ), + ( + Command::ZScore("k".into(), "m".into()), + Expect::Keys(&["k"]), + ), + ( + Command::ZMScore("k".into(), vec!["m".into()]), + Expect::Keys(&["k"]), + ), + (Command::ZRank("k".into(), "m".into()), Expect::Keys(&["k"])), + ( + Command::ZRevRank("k".into(), "m".into()), + Expect::Keys(&["k"]), + ), + ( + Command::ZRem("k".into(), vec!["m".into()]), + Expect::Keys(&["k"]), + ), + (Command::ZCard("k".into()), Expect::Keys(&["k"])), + ( + Command::ZIncrBy("k".into(), 1.0, "m".into()), + Expect::Keys(&["k"]), + ), + ( + Command::ZCount("k".into(), "0".into(), "1".into()), + Expect::Keys(&["k"]), + ), + ( + Command::ZScan("k".into(), ScanArgs::default()), + Expect::Keys(&["k"]), + ), + ( + Command::JSet("k".into(), "$".into(), "1".into()), + Expect::Keys(&["k"]), + ), + (Command::JGet("k".into(), None), Expect::Keys(&["k"])), + ( + Command::JMerge("k".into(), "{}".into()), + Expect::Keys(&["k"]), + ), + (Command::RlSet("k".into(), 1, 1), Expect::Keys(&["k"])), + (Command::RlCheck("k".into(), None), Expect::Keys(&["k"])), + (Command::Multi, Expect::KeyLess), + (Command::Exec, Expect::KeyLess), + (Command::Discard, Expect::KeyLess), + (Command::Subscribe(vec!["ch".into()]), Expect::KeyLess), + (Command::Unsubscribe(vec!["ch".into()]), Expect::KeyLess), + (Command::PSubscribe(vec!["ch".into()]), Expect::KeyLess), + (Command::PUnsubscribe(vec!["ch".into()]), Expect::KeyLess), + (Command::Publish("ch".into(), "m".into()), Expect::KeyLess), + (Command::Watch(vec!["k".into()]), Expect::Keys(&["k"])), + (Command::Unwatch(vec!["k".into()]), Expect::Keys(&["k"])), + (Command::Sync(vec![]), Expect::KeyLess), + (Command::QSub("p:*".into()), Expect::KeyLess), + (Command::QUnsub(None), Expect::KeyLess), + (Command::Save, Expect::Admin), + (Command::BgSave, Expect::Admin), + (Command::LastSave, Expect::Admin), + (Command::ReplicaOfNoOne, Expect::Admin), + (Command::Quit, Expect::KeyLess), + (Command::Client(vec!["ID".into()]), Expect::KeyLess), + ( + Command::Config(vec!["GET".into(), "*".into()]), + Expect::Admin, + ), + (Command::CommandQuery(vec![]), Expect::KeyLess), + (Command::Cluster(vec!["INFO".into()]), Expect::KeyLess), + (Command::Module(vec!["LIST".into()]), Expect::KeyLess), + (Command::Memory(vec!["DOCTOR".into()]), Expect::KeyLess), + (Command::MemoryUsage("k".into()), Expect::Keys(&["k"])), + (Command::PubSub(vec!["CHANNELS".into()]), Expect::Admin), + (Command::Unknown("X".into()), Expect::KeyLess), + ] +} + +#[test] +fn every_command_is_classified_for_scope_enforcement() { + for (cmd, expect) in all_commands() { + match (command_scope(&cmd), &expect) { + (CommandScope::KeyLess, Expect::KeyLess) => {} + (CommandScope::Admin, Expect::Admin) => {} + (CommandScope::Keys(got), Expect::Keys(want)) => { + for k in *want { + assert!( + got.contains(&k.to_string()), + "{cmd:?} must scope-check key '{k}', reported {got:?}" + ); + } + } + (got, _) => panic!("{cmd:?} classified as {got:?}, which is not what it touches"), + } + } +} + +#[test] +fn every_key_writing_command_is_classified_as_a_write() { + // `is_write_command` is a `matches!` list, which — unlike a `match` — + // has no exhaustiveness check: a new variant silently defaults to "not + // a write" and is then never replicated, logged to AOF, or broadcast. + // `primary_keys` reports the keys a command *writes*, so anything it + // names must also be classified as a write. This cross-check is what + // makes the missing entry impossible to ship. + for (cmd, _) in all_commands() { + if !primary_keys(&cmd).is_empty() { + assert!( + is_write_command(&cmd), + "{cmd:?} writes keys but is_write_command() says otherwise — \ + it would never reach replicas, the AOF, or live queries" + ); + } + } +} + +#[test] +fn eset_is_a_write_and_reports_its_key() { + let cmd = Command::ESet("presence:1".into(), "on".into()); + assert!(is_write_command(&cmd)); + assert_eq!(primary_keys(&cmd), vec!["presence:1".to_string()]); + assert_eq!(command_name(&cmd), "eset"); + // Scoped connections must not be able to write presence keys outside + // their grant. + assert!(matches!(command_scope(&cmd), CommandScope::Keys(_))); +} + +#[test] +fn eset_replays_to_replicas_as_a_plain_set() { + // A replica has no connection to scope the lifetime to, so it stores an + // ordinary key; the owning server broadcasts the DEL on disconnect. + let frame = broadcast_for( + &Command::ESet("presence:1".into(), "on".into()), + &Value::SimpleString("OK".into()), + 0, + ) + .expect("ESET must broadcast"); + let frame = String::from_utf8_lossy(&frame).into_owned(); + assert!(frame.contains("SET"), "{frame}"); + assert!(frame.contains("presence:1")); + assert!( + !frame.contains("ESET"), + "replica should receive SET, not ESET" + ); +} + +#[test] +fn every_command_is_in_the_catalog() { + // The other half of the loop closed in `catalog_names_are_real_commands`: + // every `Command` variant the parser can produce must have a catalog + // row, or `COMMAND DOCS` silently under-reports what the server can do + // and `COMMAND COUNT` lies about how much. + for (cmd, _) in all_commands() { + let name = command_name(&cmd); + if name == "unknown" { + continue; // Not a command — the reply for anything unrecognised. + } + assert!( + catalog::lookup(name).is_some(), + "{name} is a real command with no catalog entry" + ); + } +} + +#[test] +fn command_count_matches_the_catalog() { + let Value::Integer(n) = handle_command_query(&["COUNT".to_string()], 2) else { + panic!("COMMAND COUNT must reply an integer") + }; + assert_eq!(n as usize, catalog::CATALOG.len()); +} + +#[test] +fn command_info_reports_ten_fields_per_entry() { + // Redis 7 returns ten elements. A client that indexes past the sixth + // must find an empty list, not a short array. + let reply = handle_command_query(&["INFO".into(), "get".into()], 2); + let Value::Array(Some(entries)) = reply else { + panic!("expected an array") + }; + let Value::Array(Some(fields)) = &entries[0] else { + panic!("expected an entry array") + }; + assert_eq!(fields.len(), 10); + assert_eq!(fields[0], Value::BulkString(Some(b"get".to_vec()))); + assert_eq!(fields[1], Value::Integer(2)); + assert_eq!(fields[3], Value::Integer(1), "GET's key is at position 1"); +} + +#[test] +fn command_info_nils_unknown_names_in_place() { + let reply = handle_command_query(&["INFO".into(), "get".into(), "nosuchthing".into()], 2); + let Value::Array(Some(entries)) = reply else { + panic!("expected an array") + }; + assert_eq!(entries.len(), 2, "the reply stays aligned with the request"); + assert_eq!(entries[1], Value::Array(None)); +} + +#[test] +fn command_docs_shape_follows_the_protocol() { + // RESP3 gets a map; RESP2 gets the same pairs flattened. + let resp3 = handle_command_query(&["DOCS".into(), "getrange".into()], 3); + assert!(matches!(resp3, Value::Map(_)), "{resp3:?}"); + let resp2 = handle_command_query(&["DOCS".into(), "getrange".into()], 2); + let Value::Array(Some(flat)) = resp2 else { + panic!("RESP2 must flatten the map") + }; + assert_eq!(flat.len(), 2, "one command, one entry"); + assert_eq!(flat[0], Value::BulkString(Some(b"getrange".to_vec()))); +} + +#[test] +fn command_docs_omits_unknown_names() { + let Value::Array(Some(flat)) = handle_command_query(&["DOCS".into(), "nosuchthing".into()], 2) + else { + panic!("expected an array") + }; + assert!(flat.is_empty(), "an unknown name has no entry to key"); +} + +#[test] +fn command_rejects_unknown_subcommands() { + let reply = handle_command_query(&["GETKEYS".into(), "get".into(), "k".into()], 2); + assert!( + matches!(&reply, Value::Error(e) if e.contains("Unknown subcommand")), + "{reply:?}" + ); +} + +#[test] +fn client_setinfo_is_recorded_and_visible() { + let mut meta = ClientMeta::new(42, "127.0.0.1:1".into(), "127.0.0.1:6379".into()); + assert_eq!( + handle_client_command( + &["SETINFO".into(), "LIB-NAME".into(), "node-redis".into()], + &mut meta + ), + Value::SimpleString("OK".into()) + ); + assert_eq!( + handle_client_command( + &["SETINFO".into(), "LIB-VER".into(), "6.2.0".into()], + &mut meta + ), + Value::SimpleString("OK".into()) + ); + let Value::BulkString(Some(line)) = handle_client_command(&["INFO".into()], &mut meta) else { + panic!("CLIENT INFO must reply a bulk string") + }; + let line = String::from_utf8(line).unwrap(); + assert!(line.contains("lib-name=node-redis"), "{line}"); + assert!(line.contains("lib-ver=6.2.0"), "{line}"); + assert!(line.contains("id=42"), "{line}"); +} + +#[test] +fn client_setinfo_rejects_unknown_attributes() { + let mut meta = ClientMeta::new(1, String::new(), String::new()); + let reply = handle_client_command( + &["SETINFO".into(), "LIB-COLOUR".into(), "blue".into()], + &mut meta, + ); + assert!( + matches!(&reply, Value::Error(e) if e.contains("Unrecognized")), + "{reply:?}" + ); +} + +#[test] +fn client_setname_round_trips_and_rejects_spaces() { + let mut meta = ClientMeta::new(1, String::new(), String::new()); + assert_eq!( + handle_client_command(&["GETNAME".into()], &mut meta), + Value::BulkString(None), + "an unnamed connection reports nil, not an empty string" + ); + handle_client_command(&["SETNAME".into(), "worker-3".into()], &mut meta); + assert_eq!( + handle_client_command(&["GETNAME".into()], &mut meta), + Value::BulkString(Some(b"worker-3".to_vec())) + ); + // A space would break the key=value line CLIENT LIST emits. + let reply = handle_client_command(&["SETNAME".into(), "two words".into()], &mut meta); + assert!(matches!(reply, Value::Error(_)), "{reply:?}"); +} + +#[test] +fn client_declines_what_it_cannot_do() { + // KILL must not answer +OK: a caller would believe a connection had + // been closed when it is still open. + let mut meta = ClientMeta::new(1, String::new(), String::new()); + for args in [ + vec!["KILL".to_string(), "id".into(), "3".into()], + vec!["NO-EVICT".to_string(), "on".into()], + vec!["UNPAUSE".to_string()], + ] { + let reply = handle_client_command(&args, &mut meta); + assert!( + matches!(&reply, Value::Error(e) if e.contains("Unknown subcommand")), + "{args:?} -> {reply:?}" + ); + } +} + +#[test] +fn config_get_reports_values_in_force() { + let store = KeyValueStore::new(); + let facts = test_facts(); + let Value::Array(Some(flat)) = + handle_config_command(&["GET".into(), "maxmemory-policy".into()], &facts, &store) + else { + panic!("CONFIG GET must reply an array") + }; + assert_eq!(flat.len(), 2); + assert_eq!( + flat[0], + Value::BulkString(Some(b"maxmemory-policy".to_vec())) + ); + assert_eq!( + flat[1], + Value::BulkString(Some( + eviction_policy_name(store.eviction_policy()) + .as_bytes() + .to_vec() + )), + "the reported policy must be the one actually in force" + ); +} + +#[test] +fn config_get_matches_globs_and_multiple_names() { + let store = KeyValueStore::new(); + let facts = test_facts(); + let Value::Array(Some(flat)) = + handle_config_command(&["GET".into(), "maxmemory*".into()], &facts, &store) + else { + panic!("expected an array") + }; + // maxmemory and maxmemory-policy both match; the reply is flat pairs. + assert_eq!(flat.len(), 4, "{flat:?}"); + + let Value::Array(Some(none)) = + handle_config_command(&["GET".into(), "nosuchparam".into()], &facts, &store) + else { + panic!("expected an array") + }; + assert!( + none.is_empty(), + "an unmatched name yields no pair, not an error" + ); +} + +#[test] +fn config_get_masks_the_password() { + let store = KeyValueStore::new(); + let mut facts = test_facts(); + facts.auth_enabled = true; + let Value::Array(Some(flat)) = + handle_config_command(&["GET".into(), "requirepass".into()], &facts, &store) + else { + panic!("expected an array") + }; + assert_eq!( + flat[1], + Value::BulkString(Some(b"*".to_vec())), + "the password itself must never leave the process" + ); +} + +#[test] +fn config_set_refuses_rather_than_pretending() { + let store = KeyValueStore::new(); + let facts = test_facts(); + let reply = handle_config_command( + &["SET".into(), "maxmemory".into(), "100mb".into()], + &facts, + &store, + ); + // Nothing in the running server can change, so +OK would be a lie the + // operator only discovers when the limit fails to apply. + assert!( + matches!(&reply, Value::Error(e) if e.contains("configured at startup")), + "{reply:?}" + ); +} + +#[test] +fn every_command_has_a_metrics_label() { + for (cmd, _) in all_commands() { + let name = command_name(&cmd); + assert!(!name.is_empty(), "{cmd:?} has an empty metrics label"); + assert_eq!( + name, + name.to_lowercase(), + "{cmd:?} label '{name}' must be lowercase for Prometheus" + ); + } +} + +#[test] +fn primary_keys_reports_writes_only() { + // `primary_keys` answers "what did this command *write*", for + // replication and push targeting — it is deliberately NOT the + // authorization function (that is `command_scope`). Reads report + // nothing because there is no mutation to broadcast. + for cmd in [ + Command::Get("k".into()), + Command::Exists(vec!["k".into()]), + Command::Ttl("k".into()), + Command::LRange("k".into(), 0, -1), + Command::SMembers("k".into()), + Command::HGetAll("k".into()), + ] { + assert!( + primary_keys(&cmd).is_empty(), + "{cmd:?} is a read and must not be broadcast as a mutation" + ); + } + + // Writes must report every key they touch, or a replica or subscribed + // browser silently misses the change. + let writes: Vec<(Command, &[&str])> = vec![ + ( + Command::Set("k".into(), "v".into(), SetOptions::default()), + &["k"], + ), + (Command::Del(vec!["a".into(), "b".into()]), &["a", "b"]), + (Command::Incr("k".into()), &["k"]), + (Command::Rename("src".into(), "dst".into()), &["src", "dst"]), + ( + Command::SMove("src".into(), "dst".into(), "m".into()), + &["src", "dst"], + ), + ( + Command::MSet(vec![("a".into(), "1".into()), ("b".into(), "2".into())]), + &["a", "b"], + ), + ( + Command::SInterStore("dst".into(), vec!["a".into()]), + &["dst"], + ), + (Command::LPush("k".into(), vec!["v".into()]), &["k"]), + ( + Command::HSet("k".into(), vec![("f".into(), "v".into())]), + &["k"], + ), + (Command::JMerge("k".into(), "{}".into()), &["k"]), + ]; + for (cmd, want) in writes { + let got = primary_keys(&cmd); + for k in want { + assert!( + got.contains(&k.to_string()), + "{cmd:?}: primary_keys missed written key '{k}', got {got:?}" + ); + } + } +} + +// ── Pub/Sub pattern matching ────────────────────────────────────────────── + +#[test] +fn psubscribe_pattern_matching_is_not_exponential() { + // PSUBSCRIBE patterns are attacker-controlled, and every PUBLISH is + // matched against every registered pattern. This file previously used a + // recursive matcher that backtracked exponentially: a 10-wildcard + // pattern against a 36-character channel took ~7 s, so one subscriber + // could stall pub/sub for everyone. It now shares core-engine's DP + // matcher (verified equivalent). This test fails loudly if that + // regresses. + let pattern = "*a*a*a*a*a*a*a*a*a*a*b"; + let channel = "a".repeat(200); + let start = std::time::Instant::now(); + assert!(!core_engine::store::glob_match(pattern, &channel)); + assert!( + start.elapsed() < std::time::Duration::from_millis(500), + "pattern matching took {:?} — exponential backtracking is back", + start.elapsed() + ); +} + +#[test] +fn pubsub_patterns_match_the_expected_channels() { + for (pat, ch, want) in [ + ("news.*", "news.tech", true), + ("news.*", "news.", true), + ("news.*", "sports.tech", false), + ("*", "anything", true), + ("user.?", "user.1", true), + ("user.?", "user.42", false), + ] { + assert_eq!( + core_engine::store::glob_match(pat, ch), + want, + "pattern {pat:?} vs channel {ch:?}" + ); + } +} + +// ── Introspection: PUBSUB / CLUSTER / MODULE / MEMORY ───────────────────── + +/// A hub with `channels` subscribed and `patterns` psubscribed. The senders +/// are kept alive by the returned vector — dropping them would close the +/// receivers and make the hub look empty. +fn hub_with( + channels: &[(u64, &str)], + patterns: &[(u64, &str)], +) -> (PubSubHub, Vec>) { + let mut hub = PubSubHub::new(); + let mut keepalive = Vec::new(); + for (id, ch) in channels { + let (tx, rx) = mpsc::unbounded_channel(); + hub.subscribe(*id, ch, tx); + keepalive.push(rx); + } + for (id, pat) in patterns { + let (tx, rx) = mpsc::unbounded_channel(); + hub.psubscribe(*id, pat, tx); + keepalive.push(rx); + } + (hub, keepalive) +} + +fn bulk_strings(v: &Value) -> Vec { + match v { + Value::Array(Some(items)) => items + .iter() + .map(|i| match i { + Value::BulkString(Some(b)) => String::from_utf8_lossy(b).into_owned(), + other => panic!("expected a bulk string, got {other:?}"), + }) + .collect(), + other => panic!("expected an array, got {other:?}"), + } +} + +#[test] +fn pubsub_channels_lists_only_channels_with_subscribers() { + let (hub, _keep) = hub_with(&[(1, "news"), (2, "news"), (3, "sports")], &[(4, "news.*")]); + + let mut all = bulk_strings(&handle_pubsub_command(&["CHANNELS".into()], &hub)); + all.sort(); + assert_eq!(all, vec!["news".to_string(), "sports".to_string()]); + + // A pattern subscriber is not a channel. Redis reports `news.*` under + // NUMPAT and never under CHANNELS, because nobody is subscribed to a + // channel by that name. + assert!(!all.contains(&"news.*".to_string())); + + let filtered = bulk_strings(&handle_pubsub_command( + &["CHANNELS".into(), "spo*".into()], + &hub, + )); + assert_eq!(filtered, vec!["sports".to_string()]); +} + +#[test] +fn pubsub_numsub_counts_per_channel_and_keeps_the_caller_s_order() { + let (hub, _keep) = hub_with(&[(1, "news"), (2, "news"), (3, "sports")], &[(4, "news.*")]); + + let reply = handle_pubsub_command( + &[ + "NUMSUB".into(), + "sports".into(), + "news".into(), + "nobody-here".into(), + ], + &hub, + ); + assert_eq!( + reply, + Value::Array(Some(vec![ + Value::BulkString(Some(b"sports".to_vec())), + Value::Integer(1), + Value::BulkString(Some(b"news".to_vec())), + // Two subscribers, and the `news.*` pattern subscriber is not + // one of them: NUMPAT's job, counted here would be double. + Value::Integer(2), + Value::BulkString(Some(b"nobody-here".to_vec())), + // Present with a zero rather than omitted, so a caller can read + // the reply by position against the channels it asked about. + Value::Integer(0), + ])) + ); + + // No channels named is a legal call and an empty reply, not an error. + assert_eq!( + handle_pubsub_command(&["NUMSUB".into()], &hub), + Value::Array(Some(vec![])) + ); +} + +#[test] +fn pubsub_numpat_counts_distinct_patterns_not_subscribers() { + let (hub, _keep) = hub_with(&[], &[(1, "news.*"), (2, "news.*"), (3, "sports.*")]); + assert_eq!( + handle_pubsub_command(&["NUMPAT".into()], &hub), + Value::Integer(2), + "two clients on one pattern are one pattern" + ); +} + +#[test] +fn pubsub_channels_forgets_a_channel_once_its_last_subscriber_leaves() { + let (mut hub, _keep) = hub_with(&[(1, "news"), (2, "news")], &[]); + hub.unsubscribe(1, "news"); + assert_eq!( + bulk_strings(&handle_pubsub_command(&["CHANNELS".into()], &hub)), + vec!["news".to_string()] + ); + hub.unsubscribe(2, "news"); + assert!( + bulk_strings(&handle_pubsub_command(&["CHANNELS".into()], &hub)).is_empty(), + "an abandoned channel is not an active channel" + ); +} + +#[test] +fn pubsub_refuses_the_sharded_subcommands() { + let (hub, _keep) = hub_with(&[], &[]); + // A standalone redis-server answers these with an empty array, and this + // is the one place Recached deliberately does not match it: there, the + // empty array sits next to a working SSUBSCRIBE. Here there is none, so + // "no shard channels are subscribed" would invite a call that fails. + for sub in ["SHARDCHANNELS", "SHARDNUMSUB"] { + assert!( + matches!( + handle_pubsub_command(&[sub.to_string()], &hub), + Value::Error(_) + ), + "{sub} should be refused" + ); + } +} + +#[test] +fn cluster_is_refused_the_way_a_standalone_redis_refuses_it() { + // Verified against redis-server 7.2.5: a server not started in cluster + // mode rejects the whole CLUSTER container with this sentence. It does + // *not* answer INFO with cluster_enabled:0 — that lives in `INFO`. + for sub in ["INFO", "NODES", "SLOTS", "MYID", "SHARDS"] { + assert_eq!( + handle_cluster_command(&[sub.to_string()]), + Value::Error("ERR This instance has cluster support disabled".to_string()), + "CLUSTER {sub}" + ); + } +} + +#[test] +fn info_publishes_the_cluster_flag_that_cluster_info_cannot() { + let store = KeyValueStore::new(); + let body = render_info( + &["cluster".to_string()], + server_facts(), + &store, + sampled_keyspace(&store), + false, + ReplInfo::default(), + 0, + 0, + 0, + ); + assert!(body.contains("# Cluster\r\n"), "section header: {body:?}"); + assert!(body.contains("cluster_enabled:0"), "{body:?}"); + + // And it is in the default set, so a client that sends a bare INFO — + // which is what every cluster-aware client actually sends — sees it. + let default = render_info( + &[], + server_facts(), + &store, + sampled_keyspace(&store), + false, + ReplInfo::default(), + 0, + 0, + 0, + ); + assert!(default.contains("cluster_enabled:0"), "{default:?}"); +} + +#[test] +fn module_list_is_empty_and_loading_is_refused() { + assert_eq!( + handle_module_command(&["LIST".to_string()]), + Value::Array(Some(vec![])), + "no modules is an answer, not an error" + ); + for sub in ["LOAD", "LOADEX", "UNLOAD"] { + assert!( + matches!( + handle_module_command(&[sub.to_string(), "/tmp/x.so".to_string()]), + Value::Error(_) + ), + "MODULE {sub} should be refused rather than answered +OK" + ); + } +} + +#[test] +fn memory_allocator_subcommands_are_refused_with_a_reason() { + for sub in ["DOCTOR", "STATS", "PURGE", "MALLOC-STATS"] { + let Value::Error(msg) = handle_memory_command(&[sub.to_string()]) else { + panic!("MEMORY {sub} should be refused"); + }; + assert!( + msg.contains("MEMORY USAGE"), + "the refusal should name what does work: {msg}" + ); + } + assert!(matches!( + handle_memory_command(&["HELP".to_string()]), + Value::Array(Some(_)) + )); +} + +// ── Wire encoding ───────────────────────────────────────────────────────── + +#[test] +fn pubsub_message_encodes_as_a_resp3_push_frame() { + let bytes = encode_pubsub_msg( + PubSubMsg::Message { + channel: "news".into(), + message: "hello".into(), + }, + 3, + ); + let text = String::from_utf8_lossy(&bytes); + assert!( + text.starts_with('>'), + "must be a RESP3 Push frame: {text:?}" + ); + assert!(text.contains("message")); + assert!(text.contains("news")); + assert!(text.contains("hello")); +} + +#[test] +fn pubsub_message_encodes_as_an_array_for_resp2() { + // RESP2 has no push type. Sending `>` to a RESP2 client — which is + // every client that has not sent HELLO 3 — is unparseable, so a + // subscribed connection would break outright. + let bytes = encode_pubsub_msg( + PubSubMsg::Message { + channel: "news".into(), + message: "hello".into(), + }, + 2, + ); + let text = String::from_utf8_lossy(&bytes); + assert!( + text.starts_with("*3\r\n"), + "RESP2 delivery must be a 3-element array: {text:?}" + ); + assert!(!text.contains('>'), "no push frame on RESP2: {text:?}"); +} + +#[test] +fn pattern_message_carries_the_matching_pattern() { + // A pmessage must name the pattern that matched, or a client + // subscribed to several patterns cannot tell them apart. + for protover in [2u8, 3u8] { + let bytes = encode_pubsub_msg( + PubSubMsg::PMessage { + pattern: "news.*".into(), + channel: "news.tech".into(), + message: "hi".into(), + }, + protover, + ); + let text = String::from_utf8_lossy(&bytes); + assert!(text.contains("pmessage"), "protover {protover}"); + assert!(text.contains("news.*"), "protover {protover}"); + assert!(text.contains("news.tech"), "protover {protover}"); + } +} + +// ── HELLO / protocol negotiation ───────────────────────────────────────── + +#[test] +fn hello_defaults_to_the_connections_current_version() { + // Bare HELLO reports, it does not change. A client using it purely to + // read server info must not be silently switched to another protocol. + let mut protover = 2u8; + let bytes = process_hello(None, &mut protover, true, false); + assert_eq!(protover, 2, "bare HELLO must not change the version"); + let text = String::from_utf8_lossy(&bytes); + assert!( + text.starts_with('*'), + "RESP2 reply must be an array: {text:?}" + ); + assert!(text.contains("recached")); + assert!(text.contains(":2\r\n"), "proto must report 2: {text:?}"); +} + +#[test] +fn hello_3_upgrades_and_replies_with_a_map() { + let mut protover = 2u8; + let bytes = process_hello(Some("3"), &mut protover, true, false); + assert_eq!(protover, 3); + let text = String::from_utf8_lossy(&bytes); + assert!(text.starts_with("%6\r\n"), "must be a 6-pair map: {text:?}"); + assert!(text.contains(":3\r\n"), "proto must report 3: {text:?}"); +} + +#[test] +fn hello_3_then_2_downgrades_again() { + let mut protover = 2u8; + process_hello(Some("3"), &mut protover, true, false); + assert_eq!(protover, 3); + let bytes = process_hello(Some("2"), &mut protover, true, false); + assert_eq!(protover, 2, "HELLO 2 must downgrade"); + assert!(String::from_utf8_lossy(&bytes).starts_with('*')); +} + +#[test] +fn hello_rejects_unsupported_versions_without_changing_protocol() { + // A client probing for a version the server does not speak must get a + // clean NOPROTO and stay on what it had — not be left in a half-state. + for bad in ["4", "1", "0", "abc", "", "255", "-1", "3.0"] { + let mut protover = 2u8; + let bytes = process_hello(Some(bad), &mut protover, true, false); + let text = String::from_utf8_lossy(&bytes); + assert!( + text.starts_with("-NOPROTO"), + "HELLO {bad:?} must be refused: {text:?}" + ); + assert_eq!(protover, 2, "HELLO {bad:?} must not change the version"); + } +} + +#[test] +fn hello_does_not_leak_server_details_before_auth() { + let mut protover = 2u8; + let bytes = process_hello(Some("3"), &mut protover, false, false); + let text = String::from_utf8_lossy(&bytes); + assert!(text.starts_with("-NOAUTH"), "{text:?}"); + assert!( + !text.contains("recached") && !text.contains(env!("CARGO_PKG_VERSION")), + "unauthenticated HELLO must not fingerprint the server: {text:?}" + ); +} + +#[test] +fn hello_reports_replica_role() { + let mut protover = 3u8; + let primary = + String::from_utf8_lossy(&process_hello(None, &mut protover, true, false)).into_owned(); + let replica = + String::from_utf8_lossy(&process_hello(None, &mut protover, true, true)).into_owned(); + assert!(primary.contains("master"), "{primary:?}"); + assert!(replica.contains("replica"), "{replica:?}"); +} + +// ── INFO ────────────────────────────────────────────────────────────────── + +fn test_facts() -> ServerFacts { + ServerFacts { + start: SystemTime::now() - std::time::Duration::from_secs(90_000), + run_id: "a".repeat(40), + tcp_port: 6379, + ws_port: 6380, + max_connections: 512, + tls_enabled: true, + auth_enabled: true, + aof_enabled: true, + } +} + +/// Render `sections` against `store`, walking it for the keyspace numbers. +/// +/// Tests pass the sample explicitly rather than going through the shared +/// 5s cache — `render_info` is pure so that concurrent tests cannot +/// observe each other's keyspace through a process-global. +fn info_for(sections: &[&str], store: &KeyValueStore) -> String { + render_info( + §ions.iter().map(|s| s.to_string()).collect::>(), + &test_facts(), + store, + store.keyspace_sample(), + false, + ReplInfo::default(), + 1_700_000_000, + 0, + 0, + ) +} + +/// Parse an INFO payload into (section, field) → value, enforcing the shape +/// clients rely on: CRLF endings, `# Section` headers, `field:value` lines. +fn parse_info(payload: &str) -> HashMap<(String, String), String> { + assert!( + !payload.contains('\n') || payload.contains("\r\n"), + "INFO must use CRLF line endings" + ); + let mut out = HashMap::new(); + let mut section = String::new(); + for line in payload.split("\r\n") { + if line.is_empty() { + continue; + } + if let Some(name) = line.strip_prefix("# ") { + section = name.to_lowercase(); + continue; + } + let (k, v) = line + .split_once(':') + .unwrap_or_else(|| panic!("malformed INFO line: {line:?}")); + out.insert((section.clone(), k.to_string()), v.to_string()); + } + out +} + +#[test] +fn info_default_emits_every_default_section() { + let store = KeyValueStore::new(); + let payload = info_for(&[], &store); + for section in DEFAULT_INFO_SECTIONS { + let header = format!("# {}{}\r\n", section[..1].to_uppercase(), §ion[1..]); + assert!( + payload.contains(&header), + "missing section header {header:?} in {payload:?}" + ); + } +} + +#[test] +fn info_uses_crlf_and_blank_line_separated_sections() { + let store = KeyValueStore::new(); + let payload = info_for(&["server", "clients"], &store); + assert!(payload.starts_with("# Server\r\n"), "{payload:?}"); + // A blank line must close each section, or parsers merge them. + assert!(payload.contains("\r\n\r\n# Clients\r\n"), "{payload:?}"); + assert!(payload.ends_with("\r\n\r\n"), "{payload:?}"); + assert!(!payload.contains('\n') || !payload.replace("\r\n", "").contains('\n')); +} + +#[test] +fn info_server_section_reports_compat_version_separately_from_ours() { + let store = KeyValueStore::new(); + let f = parse_info(&info_for(&["server"], &store)); + // Clients feature-gate on redis_version, so it must be a Redis version, + // never Recached's own — that is the entire point of the split. + assert_eq!(f[&("server".into(), "redis_version".into())], "6.2.0"); + assert_eq!( + f[&("server".into(), "recached_version".into())], + env!("CARGO_PKG_VERSION") + ); + assert_eq!(f[&("server".into(), "redis_mode".into())], "standalone"); + assert_eq!(f[&("server".into(), "tcp_port".into())], "6379"); + assert_eq!(f[&("server".into(), "recached_ws_port".into())], "6380"); + assert_eq!(f[&("server".into(), "run_id".into())].len(), 40); + // 90_000s of uptime is one day and change. + assert_eq!(f[&("server".into(), "uptime_in_days".into())], "1"); + assert!( + f[&("server".into(), "uptime_in_seconds".into())] + .parse::() + .unwrap() + >= 90_000 + ); +} + +#[test] +fn info_memory_section_reports_limits_and_policy() { + let store = KeyValueStore::with_config(Some(50), Some(1024 * 1024), EvictionPolicy::AllKeysLru); + let f = parse_info(&info_for(&["memory"], &store)); + assert_eq!(f[&("memory".into(), "maxmemory".into())], "1048576"); + assert_eq!(f[&("memory".into(), "maxmemory_human".into())], "1.00M"); + assert_eq!( + f[&("memory".into(), "maxmemory_policy".into())], + "allkeys-lru" + ); + assert_eq!(f[&("memory".into(), "recached_max_keys".into())], "50"); +} + +#[test] +fn info_memory_reports_zero_maxmemory_when_unbounded() { + // Redis reports 0 for "no limit"; None must not leak as a debug string. + let f = parse_info(&info_for(&["memory"], &KeyValueStore::new())); + assert_eq!(f[&("memory".into(), "maxmemory".into())], "0"); + assert_eq!( + f[&("memory".into(), "maxmemory_policy".into())], + "noeviction" + ); +} + +#[test] +fn info_persistence_always_reports_loading_zero() { + // A client's ready-check gates on this field; the snapshot is loaded + // before any listener binds, so a reachable server is never loading. + let f = parse_info(&info_for(&["persistence"], &KeyValueStore::new())); + assert_eq!(f[&("persistence".into(), "loading".into())], "0"); + assert_eq!( + f[&("persistence".into(), "rdb_last_save_time".into())], + "1700000000" + ); + assert_eq!(f[&("persistence".into(), "aof_enabled".into())], "1"); +} + +#[test] +fn info_replication_reports_both_redis_and_recached_spellings() { + let store = KeyValueStore::new(); + let repl = ReplInfo { + connected: 2, + queue_depth: 7, + lag_frames: 3, + }; + let primary = parse_info(&render_info( + &[], + &test_facts(), + &store, + store.keyspace_sample(), + false, + repl, + 0, + 0, + 0, + )); + assert_eq!(primary[&("replication".into(), "role".into())], "master"); + // Tooling greps for `connected_slaves`; the modern alias ships too. + assert_eq!( + primary[&("replication".into(), "connected_slaves".into())], + "2" + ); + assert_eq!( + primary[&("replication".into(), "connected_replicas".into())], + "2" + ); + assert_eq!( + primary[&( + "replication".into(), + "recached_replication_lag_frames".into() + )], + "3" + ); + + let replica = parse_info(&render_info( + &[], + &test_facts(), + &store, + store.keyspace_sample(), + true, + ReplInfo::default(), + 0, + 0, + 0, + )); + // Redis still spells a replica `slave` in INFO, and clients match on it. + assert_eq!(replica[&("replication".into(), "role".into())], "slave"); +} + +#[test] +fn info_keyspace_omits_the_db_line_when_empty_and_counts_ttls_when_not() { + let store = KeyValueStore::new(); + assert!( + !info_for(&["keyspace"], &store).contains("db0:"), + "an empty keyspace must not report a db0 line" + ); + + store.execute(Command::Set( + "a".into(), + b"v".to_vec(), + SetOptions::default(), + )); + store.execute(Command::Set( + "b".into(), + b"v".to_vec(), + SetOptions::default(), + )); + store.execute(Command::Expire("b".into(), 60)); + let payload = info_for(&["keyspace"], &store); + assert!( + payload.contains("db0:keys=2,expires=1,avg_ttl=0"), + "{payload:?}" + ); +} + +#[test] +fn sampled_keyspace_falls_back_to_a_live_walk_before_the_sampler_runs() { + // First INFO of a process arrives before the 5s sampler has ever run, + // and must not report an empty keyspace. + let store = KeyValueStore::new(); + store.execute(Command::Set( + "k".into(), + b"v".to_vec(), + SetOptions::default(), + )); + SAMPLED_KEYS.store(u64::MAX, Ordering::Relaxed); + assert_eq!(sampled_keyspace(&store).keys, 1); +} + +#[test] +fn info_unknown_section_yields_nothing() { + // Redis answers an unknown section with an empty payload, not an error. + assert_eq!(info_for(&["nosuchsection"], &KeyValueStore::new()), ""); +} + +#[test] +fn info_all_and_everything_expand_to_the_default_sections() { + let store = KeyValueStore::new(); + let default = info_for(&[], &store); + for alias in ["all", "everything", "default"] { + assert_eq!( + info_for(&[alias], &store).lines().count(), + default.lines().count(), + "INFO {alias} must cover the default sections" + ); + } +} + +#[test] +fn info_honours_section_selection_and_order() { + let payload = info_for(&["clients", "server"], &KeyValueStore::new()); + assert!(payload.starts_with("# Clients\r\n"), "{payload:?}"); + assert!(payload.contains("# Server\r\n"), "{payload:?}"); + assert!(!payload.contains("# Memory"), "{payload:?}"); +} + +#[test] +fn human_bytes_matches_redis_formatting() { + assert_eq!(human_bytes(0), "0B"); + assert_eq!(human_bytes(512), "512B"); + assert_eq!(human_bytes(1024), "1.00K"); + assert_eq!(human_bytes(1024 * 1024), "1.00M"); + assert_eq!(human_bytes(3 * 1024 * 1024 * 1024), "3.00G"); +} + +#[test] +fn run_ids_are_forty_hex_chars_and_differ_per_process() { + let a = generate_run_id(); + assert_eq!(a.len(), 40); + assert!(a.chars().all(|c| c.is_ascii_hexdigit()), "{a}"); + assert_ne!(a, generate_run_id()); +} + +#[test] +fn info_is_administrative_scope() { + // Scoped WebSocket connections must not be able to read server-wide + // state, so INFO has to classify as Admin, not KeyLess. + assert!(matches!( + command_scope(&Command::Info(vec![])), + CommandScope::Admin + )); +} + +#[test] +fn info_is_not_a_write_command() { + assert!(!is_write_command(&Command::Info(vec![]))); +} + +#[tokio::test] +async fn info_over_tcp_returns_a_parseable_bulk_string() { + let srv = spawn_server().await; + let mut c = RespClient::connect(srv.tcp_addr).await; + + let Value::BulkString(Some(bytes)) = c.cmd(&["INFO"]).await else { + panic!("INFO must reply with a bulk string"); + }; + let payload = String::from_utf8(bytes).unwrap(); + let f = parse_info(&payload); + assert_eq!(f[&("server".into(), "redis_version".into())], "6.2.0"); + assert_eq!(f[&("replication".into(), "role".into())], "master"); + assert!(f.contains_key(&("stats".into(), "total_commands_processed".into()))); +} + +#[tokio::test] +async fn info_section_argument_is_honoured_over_the_wire() { + let srv = spawn_server().await; + let mut c = RespClient::connect(srv.tcp_addr).await; + + let Value::BulkString(Some(bytes)) = c.cmd(&["INFO", "server"]).await else { + panic!("INFO must reply with a bulk string"); + }; + let payload = String::from_utf8(bytes).unwrap(); + assert!(payload.starts_with("# Server\r\n"), "{payload:?}"); + assert!(!payload.contains("# Memory"), "{payload:?}"); +} + +#[tokio::test] +async fn info_reflects_live_server_state() { + let srv = spawn_server().await; + let mut c = RespClient::connect(srv.tcp_addr).await; + + c.cmd(&["SET", "k", "v"]).await; + c.cmd(&["GET", "k"]).await; // hit + c.cmd(&["GET", "missing"]).await; // miss + + let Value::BulkString(Some(bytes)) = c.cmd(&["INFO", "stats", "persistence"]).await else { + panic!("INFO must reply with a bulk string"); + }; + let f = parse_info(&String::from_utf8(bytes).unwrap()); + assert!( + f[&("stats".into(), "keyspace_hits".into())] + .parse::() + .unwrap() + >= 1 + ); + assert!( + f[&("stats".into(), "keyspace_misses".into())] + .parse::() + .unwrap() + >= 1 + ); + // The SET must show up as an unsaved change. + assert!( + f[&("persistence".into(), "rdb_changes_since_last_save".into())] + .parse::() + .unwrap() + >= 1 + ); +} + +#[tokio::test] +async fn info_requires_authentication() { + let srv = spawn_server_cfg(Some("hunter2"), None, false).await; + let mut c = RespClient::connect(srv.tcp_addr).await; + + // INFO leaks deployment details, so it must sit behind AUTH like every + // other non-handshake command. + match c.cmd(&["INFO"]).await { + Value::Error(e) => assert!(e.starts_with("NOAUTH"), "{e}"), + other => panic!("unauthenticated INFO must be refused, got {other:?}"), + } + + assert_eq!(c.cmd(&["AUTH", "hunter2"]).await, ok()); + assert!(matches!(c.cmd(&["INFO"]).await, Value::BulkString(Some(_)))); +} + +#[tokio::test] +async fn info_on_a_replica_reports_the_slave_role() { + let srv = spawn_server_cfg(None, None, true).await; + let mut c = RespClient::connect(srv.tcp_addr).await; + + let Value::BulkString(Some(bytes)) = c.cmd(&["INFO", "replication"]).await else { + panic!("INFO must reply with a bulk string"); + }; + let f = parse_info(&String::from_utf8(bytes).unwrap()); + assert_eq!(f[&("replication".into(), "role".into())], "slave"); +} + +#[test] +fn subscribe_ack_reports_the_running_subscription_count() { + let bytes = resp_subscribe_ack("subscribe", "news", 3); + let text = String::from_utf8_lossy(&bytes); + assert!(text.contains("subscribe")); + assert!(text.contains("news")); + assert!( + text.contains(":3"), + "count must be a RESP integer: {text:?}" + ); +} + +// ── Score formatting ────────────────────────────────────────────────────── + +#[test] +fn scores_format_without_trailing_decimals() { + // Redis returns "1" not "1.0" — clients parse these as integers. + assert_eq!(format_f64_score(1.0), "1"); + assert_eq!(format_f64_score(-5.0), "-5"); + assert_eq!(format_f64_score(0.0), "0"); + assert_eq!(format_f64_score(1.5), "1.5"); + assert_eq!(format_f64_score(-0.25), "-0.25"); +} + +#[test] +fn scores_format_infinities_as_redis_does() { + assert_eq!(format_f64_score(f64::INFINITY), "inf"); + assert_eq!(format_f64_score(f64::NEG_INFINITY), "-inf"); +} + +#[test] +fn very_large_scores_do_not_lose_their_exponent() { + // Past 1e15 the integer shortcut is skipped, because casting to i64 + // would silently truncate. + let big = 1e16_f64; + let s = format_f64_score(big); + assert!( + s.contains('e') || s.len() > 15, + "unexpected formatting: {s}" + ); +} + +// ── Save conditions ─────────────────────────────────────────────────────── + +#[test] +fn save_conditions_parse_as_seconds_colon_changes() { + let c = parse_save_conditions("900:1,300:10,60:10000"); + assert_eq!(c.len(), 3); + assert_eq!(c[0].secs, 900); + assert_eq!(c[0].changes, 1); + assert_eq!(c[2].secs, 60); + assert_eq!(c[2].changes, 10000); +} + +#[test] +fn save_conditions_tolerate_whitespace() { + let c = parse_save_conditions(" 900 : 1 , 300 : 10 "); + assert_eq!(c.len(), 2); + assert_eq!(c[0].secs, 900); + assert_eq!(c[1].changes, 10); +} + +#[test] +fn malformed_save_conditions_are_skipped_not_fatal() { + // A bad pair is dropped so one typo cannot disable autosave entirely — + // but a wholly invalid string yields no conditions, which the caller + // treats as "autosave off". + let c = parse_save_conditions("900:1,garbage,300:10"); + assert_eq!(c.len(), 2, "valid pairs survive a bad one"); + assert!(parse_save_conditions("").is_empty()); + assert!(parse_save_conditions("nonsense").is_empty()); + assert!( + parse_save_conditions("900").is_empty(), + "missing ':changes'" + ); +} + +// ── TLS configuration ───────────────────────────────────────────────────── + +#[test] +fn tls_requires_both_cert_and_key() { + assert_eq!( + resolve_tls_paths(None, None).unwrap(), + None, + "neither set → plaintext" + ); + assert_eq!( + resolve_tls_paths(Some("c.pem".into()), Some("k.pem".into())).unwrap(), + Some(("c.pem".to_string(), "k.pem".to_string())) + ); +} + +#[test] +fn tls_half_configured_is_refused_not_downgraded() { + // The dangerous case: an operator sets the cert, mistypes the key + // variable, and the server used to serve plaintext on both ports while + // reporting itself healthy. Traffic believed encrypted was not. + let cert_only = resolve_tls_paths(Some("c.pem".into()), None).unwrap_err(); + assert!(cert_only.contains("RECACHED_TLS_KEY"), "got {cert_only}"); + assert!( + cert_only.contains("plaintext"), + "must explain the risk: {cert_only}" + ); + + let key_only = resolve_tls_paths(None, Some("k.pem".into())).unwrap_err(); + assert!(key_only.contains("RECACHED_TLS_CERT"), "got {key_only}"); +} + +// ── IP allowlist ────────────────────────────────────────────────────────── + +#[test] +fn allow_ips_parses_exact_addresses() { + let ips = parse_allow_ips("10.0.1.5, 10.0.1.6").unwrap(); + assert_eq!(ips.len(), 2); + assert!(ips.contains(&IpAddr::from_str("10.0.1.5").unwrap())); + // IPv6 literals are accepted too. + let v6 = parse_allow_ips("::1").unwrap(); + assert_eq!(v6, vec![IpAddr::from_str("::1").unwrap()]); +} + +#[test] +fn allow_ips_rejects_cidr_instead_of_silently_narrowing() { + // A CIDR range used to be dropped with only a warning, leaving an + // allowlist that excluded every host the operator meant to admit. + let err = parse_allow_ips("10.0.0.0/8").unwrap_err(); + assert!(err.contains("10.0.0.0/8"), "must name the bad entry: {err}"); + assert!(err.contains("CIDR"), "must explain why: {err}"); +} + +#[test] +fn allow_ips_rejects_a_partially_valid_list() { + // One good entry must not mask a typo in another — the result would be + // a narrower allowlist than configured. + assert!(parse_allow_ips("10.0.1.5,not-an-ip").is_err()); + assert!( + parse_allow_ips("localhost").is_err(), + "hostnames unsupported" + ); +} + +#[test] +fn allow_ips_rejects_an_empty_result_that_would_block_everything() { + // An all-invalid list previously produced an empty allowlist, and an + // empty allowlist rejects every connection while the process still + // starts and passes health checks. + let err = parse_allow_ips(" ").unwrap_err(); + assert!(err.contains("reject every connection"), "got {err}"); + assert!(parse_allow_ips(",,,").is_err()); +} + +#[test] +fn allow_ips_tolerates_incidental_whitespace_and_trailing_commas() { + let ips = parse_allow_ips(" 127.0.0.1 , 10.0.0.1 ,").unwrap(); + assert_eq!(ips.len(), 2); +} + +/// `record_command` looks the label up in an immutable, pre-built map. A +/// label `command_name` can produce but the catalog does not list still +/// works — it falls back to an uncached registry lookup — but it pays that +/// lookup on every single call, so the gap should fail CI rather than +/// quietly become a hot-path cost. +#[test] +fn command_name_labels_are_all_pre_registered() { + let labels = [ + "get", + "set", + "del", + "incr", + "decr", + "exists", + "expire", + "ttl", + "type", + "append", + "strlen", + "hset", + "hget", + "hgetall", + "hdel", + "hlen", + "lpush", + "rpush", + "lpop", + "rpop", + "lrange", + "llen", + "sadd", + "srem", + "smembers", + "scard", + "zadd", + "zrange", + "zrem", + "zscore", + "ping", + "auth", + "hello", + "quit", + "client", + "config", + "command", + "scan", + "subscribe", + "publish", + "multi", + "exec", + "watch", + UNKNOWN_COMMAND, + ]; + let missing: Vec<&str> = labels + .into_iter() + .filter(|l| !CMD_COUNTERS.contains_key(l)) + .collect(); + assert!( + missing.is_empty(), + "labels with no pre-built counter (each costs a registry lookup per command): {missing:?}" + ); +} + +/// The counter table is built once from the catalog and never mutated, so +/// concurrent `record_command` calls need no lock and cannot poison one. +/// The previous `RwLock` was `.unwrap()`ed on every command: one panic while +/// holding it poisoned the lock and every later command panicked with it. +#[test] +fn record_command_is_safe_from_many_threads_at_once() { + let threads: Vec<_> = (0..8) + .map(|_| { + std::thread::spawn(|| { + for _ in 0..5_000 { + record_command("get"); + record_command("set"); + record_command(UNKNOWN_COMMAND); + } + }) + }) + .collect(); + for t in threads { + t.join().expect("record_command panicked under contention"); + } +} + +#[test] +fn command_name_is_stable_and_lowercase() { + // These strings become Prometheus label values; renaming one silently + // breaks existing dashboards and alerts. + let cases = [ + (Command::Get("k".into()), "get"), + ( + Command::Set("k".into(), "v".into(), SetOptions::default()), + "set", + ), + (Command::Del(vec!["k".into()]), "del"), + (Command::Incr("k".into()), "incr"), + (Command::HGetAll("k".into()), "hgetall"), + (Command::LPush("k".into(), vec!["v".into()]), "lpush"), + (Command::SAdd("k".into(), vec!["m".into()]), "sadd"), + (Command::Ping(None), "ping"), + ]; + for (cmd, expected) in cases { + assert_eq!(command_name(&cmd), expected, "label drift for {cmd:?}"); + } +} + +// ── Config parsing ──────────────────────────────────────────────────────── + +#[test] +fn parse_memory_bytes_accepts_units_and_bare_numbers() { + assert_eq!(parse_memory_bytes("1024"), Some(1024)); + assert_eq!(parse_memory_bytes("1kb"), Some(1024)); + assert_eq!(parse_memory_bytes("2mb"), Some(2 * 1024 * 1024)); + assert_eq!(parse_memory_bytes("1gb"), Some(1024 * 1024 * 1024)); +} + +#[test] +fn parse_memory_bytes_is_case_and_whitespace_tolerant() { + assert_eq!(parse_memory_bytes(" 2MB "), Some(2 * 1024 * 1024)); + assert_eq!(parse_memory_bytes("2 mb"), Some(2 * 1024 * 1024)); + assert_eq!(parse_memory_bytes("1Gb"), Some(1024 * 1024 * 1024)); +} + +#[test] +fn parse_memory_bytes_rejects_nonsense_rather_than_defaulting() { + // Returning None lets the caller fall back explicitly; silently + // parsing "10 bananas" as 10 bytes would cap memory at nothing. + for bad in ["", "abc", "10 bananas", "-5", "1.5mb", "mb"] { + assert_eq!(parse_memory_bytes(bad), None, "{bad:?} should not parse"); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn integration_ws_sync_scope_filters_fanout() { + let srv = spawn_ws_server().await; + let mut scoped = WsClient::connect(srv.tcp_addr).await; + let mut unscoped = WsClient::connect(srv.tcp_addr).await; + let mut writer = WsClient::connect(srv.tcp_addr).await; + + // Open mode: SYNC with literal patterns. + assert_eq!(scoped.cmd(&["SYNC", "cart:*"]).await, arr(&["cart:*"])); + + assert_eq!(writer.cmd(&["SET", "cart:1", "x"]).await, ok()); + assert_eq!(writer.cmd(&["SET", "other:1", "y"]).await, ok()); + + // Scoped client sees the cart write and nothing else. + let push = scoped.recv_push(1000).await.expect("expected cart:1 push"); + assert!(push.contains("cart:1"), "unexpected push: {push}"); + assert!( + scoped.recv_push(300).await.is_none(), + "out-of-scope push leaked to scoped client" + ); + + // Unscoped client (legacy mode) sees both. + let p1 = unscoped.recv_push(1000).await.expect("push 1"); + let p2 = unscoped.recv_push(1000).await.expect("push 2"); + assert!(p1.contains("cart:1") && p2.contains("other:1")); + + // Bare SYNC reports current scopes. + assert_eq!(scoped.cmd(&["SYNC"]).await, arr(&["cart:*"])); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn integration_ws_sync_strict_mode_gates_and_filters() { + let secret = "integration-secret"; + let srv = spawn_ws_server_cfg(Some(secret.to_string())).await; + let mut client = WsClient::connect(srv.tcp_addr).await; + + // No token yet: key commands and pushes are refused. + let r = client.cmd(&["GET", "cart:1"]).await; + assert!(matches!(&r, Value::Error(e) if e.contains("NOSCOPE"))); + // Literal patterns are rejected in strict mode. + let r = client.cmd(&["SYNC", "cart:*"]).await; + assert!(matches!(&r, Value::Error(e) if e.contains("signed scopes"))); + // Garbage token. + let r = client.cmd(&["SYNC", "TOKEN", "not-a-token"]).await; + assert!(matches!(&r, Value::Error(e) if e.contains("invalid sync token"))); + + // Valid token: scoped to cart:* only. + let tok = mint_sync_token(secret, "cart:*"); + assert_eq!(client.cmd(&["SYNC", "TOKEN", &tok]).await, arr(&["cart:*"])); + + // In-scope commands work; out-of-scope and admin are refused. + assert_eq!(client.cmd(&["SET", "cart:1", "x"]).await, ok()); + assert_eq!(client.cmd(&["GET", "cart:1"]).await, bulk("x")); + let r = client.cmd(&["GET", "secret-key"]).await; + assert!(matches!(&r, Value::Error(e) if e.contains("NOSCOPE"))); + let r = client.cmd(&["KEYS", "*"]).await; + assert!(matches!(&r, Value::Error(e) if e.contains("NOSCOPE"))); + + // Fan-out: a second scoped client writes in and out of the first's scope. + let mut writer = WsClient::connect(srv.tcp_addr).await; + let wtok = mint_sync_token(secret, "cart:*,other:*"); + assert_eq!( + writer.cmd(&["SYNC", "TOKEN", &wtok]).await, + arr(&["cart:*", "other:*"]) + ); + assert_eq!(writer.cmd(&["SET", "cart:2", "a"]).await, ok()); + assert_eq!(writer.cmd(&["SET", "other:2", "b"]).await, ok()); + + let push = client.recv_push(1000).await.expect("expected cart:2 push"); + assert!(push.contains("cart:2"), "unexpected push: {push}"); + assert!( + client.recv_push(300).await.is_none(), + "out-of-scope push leaked on strict connection" + ); +} + +// ── Exactly-once delivery (DEDUP) ───────────────────────────────────────── + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn integration_dedup_skips_replayed_writes() { + let srv = spawn_ws_server().await; + let mut c = WsClient::connect(srv.tcp_addr).await; + + // First delivery applies. + assert_eq!( + c.cmd(&["DEDUP", "client-a", "1", "INCRBY", "n", "2"]).await, + int(2) + ); + // Exact replay (ack lost, client re-sent) is skipped. + assert_eq!( + c.cmd(&["DEDUP", "client-a", "1", "INCRBY", "n", "2"]).await, + Value::SimpleString("DUP".into()) + ); + // Higher id applies. + assert_eq!( + c.cmd(&["DEDUP", "client-a", "2", "INCRBY", "n", "3"]).await, + int(5) + ); + // The high-water mark survives a reconnect — the whole point. + let mut c2 = WsClient::connect(srv.tcp_addr).await; + assert_eq!( + c2.cmd(&["DEDUP", "client-a", "2", "INCRBY", "n", "3"]) + .await, + Value::SimpleString("DUP".into()) + ); + assert_eq!(srv.store.execute(Command::Get("n".into())), bulk("5")); + // A different client id has an independent mark. + assert_eq!( + c2.cmd(&["DEDUP", "client-b", "1", "INCRBY", "n", "1"]) + .await, + int(6) + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn integration_dedup_respects_sync_scopes() { + let secret = "dedup-secret"; + let srv = spawn_ws_server_cfg(Some(secret.to_string())).await; + let mut c = WsClient::connect(srv.tcp_addr).await; + let tok = mint_sync_token(secret, "cart:*"); + assert_eq!(c.cmd(&["SYNC", "TOKEN", &tok]).await, arr(&["cart:*"])); + + // Scope enforcement applies to the wrapped command. + assert_eq!( + c.cmd(&["DEDUP", "c1", "1", "SET", "cart:1", "x"]).await, + ok() + ); + let r = c.cmd(&["DEDUP", "c1", "2", "SET", "admin:1", "x"]).await; + assert!(matches!(&r, Value::Error(e) if e.contains("NOSCOPE"))); +} + +// ── JSON over the wire ──────────────────────────────────────────────────── + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn integration_json_commands_and_fanout() { + let srv = spawn_ws_server().await; + let mut writer = WsClient::connect(srv.tcp_addr).await; + let mut peer = WsClient::connect(srv.tcp_addr).await; + + assert_eq!( + writer.cmd(&["JSET", "doc:1", "$", r#"{"a":1}"#]).await, + ok() + ); + assert_eq!(writer.cmd(&["JGET", "doc:1", "$.a"]).await, bulk("1")); + assert_eq!( + writer + .cmd(&["JMERGE", "doc:1", r#"{"b":2,"a":null}"#]) + .await, + ok() + ); + assert_eq!(writer.cmd(&["JGET", "doc:1"]).await, bulk(r#"{"b":2}"#)); + + // Peers receive the writes as replayable pushes. + let p = peer.recv_push(1000).await.expect("JSET push"); + assert!(p.contains("JSET") && p.contains("doc:1"), "push: {p}"); + let p2 = peer.recv_push(1000).await.expect("JMERGE push"); + assert!(p2.contains("JMERGE"), "push: {p2}"); + + // Failed writes are not broadcast. + let r = writer.cmd(&["JSET", "doc:1", "$", "{bad"]).await; + assert!(matches!(&r, Value::Error(e) if e.contains("invalid JSON"))); + assert!(peer.recv_push(300).await.is_none()); +} + +// ── Live queries (QSUB / QUNSUB) ────────────────────────────────────────── + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn integration_ws_qsub_initial_state_and_diffs() { + let srv = spawn_ws_server().await; + let mut writer = WsClient::connect(srv.tcp_addr).await; + let mut client = WsClient::connect(srv.tcp_addr).await; + + // Pre-existing state the subscription must deliver up front. + assert_eq!(writer.cmd(&["SET", "cart:1", "apples"]).await, ok()); + assert_eq!(writer.cmd(&["SET", "other:1", "zzz"]).await, ok()); + + let initial = client.cmd(&["QSUB", "cart:*"]).await; + match &initial { + Value::Array(Some(items)) => { + assert_eq!( + items.len(), + 4, + "expected tag + pattern + one pair: {items:?}" + ); + assert_eq!(items[0], bulk("qstate")); + assert_eq!(items[1], bulk("cart:*")); + assert_eq!(items[2], bulk("cart:1")); + assert_eq!(items[3], bulk("apples")); + } + other => panic!("expected initial-state array, got {other:?}"), + } + + // A matching write arrives as a keychange diff… + assert_eq!(writer.cmd(&["SET", "cart:2", "pears"]).await, ok()); + let (key, value) = client.recv_keychange(1000).await.expect("cart:2 diff"); + assert_eq!((key.as_str(), &value), ("cart:2", &bulk("pears"))); + + // …a non-matching write does not… + assert_eq!(writer.cmd(&["SET", "other:2", "yyy"]).await, ok()); + assert!(client.recv_keychange(300).await.is_none()); + + // …a deletion arrives as a nil keychange… + assert_eq!(writer.cmd(&["DEL", "cart:2"]).await, int(1)); + let (key, value) = client.recv_keychange(1000).await.expect("delete diff"); + assert_eq!((key.as_str(), &value), ("cart:2", &nil())); + + // …and QUNSUB stops the stream. + assert_eq!(client.cmd(&["QUNSUB", "cart:*"]).await, ok()); + assert_eq!(writer.cmd(&["SET", "cart:3", "plums"]).await, ok()); + assert!(client.recv_keychange(300).await.is_none()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn integration_ws_qsub_strict_scope() { + let secret = "qsub-secret"; + let srv = spawn_ws_server_cfg(Some(secret.to_string())).await; + let mut client = WsClient::connect(srv.tcp_addr).await; + + let tok = mint_sync_token(secret, "cart:*"); + assert_eq!(client.cmd(&["SYNC", "TOKEN", &tok]).await, arr(&["cart:*"])); + + // A narrower pattern under the grant is allowed (prefix-style cover). + assert_eq!( + client.cmd(&["QSUB", "cart:42:*"]).await, + arr(&["qstate", "cart:42:*"]) + ); + // A pattern outside the grant is refused. + let r = client.cmd(&["QSUB", "admin:*"]).await; + assert!(matches!(&r, Value::Error(e) if e.contains("NOSCOPE"))); + + // Diffs flow for the subscribed pattern. + let mut writer = WsClient::connect(srv.tcp_addr).await; + let wtok = mint_sync_token(secret, "cart:*"); + writer.cmd(&["SYNC", "TOKEN", &wtok]).await; + assert_eq!(writer.cmd(&["SET", "cart:42:item", "x"]).await, ok()); + let (key, _) = client.recv_keychange(1000).await.expect("scoped diff"); + assert_eq!(key, "cart:42:item"); +} + +// ───────────────────────────────────────────────────────────────────────────── +// Hardening: the network-exposure fixes and their decision functions. +// +// Every check here guards a boundary that was open in 0.2.4 or earlier. The +// pure-function shape is deliberate: the replication gate and the origin +// allowlist are decisions, and a decision can be asserted without standing up a +// listener or mutating process-global environment state. +// ───────────────────────────────────────────────────────────────────────────── +#[cfg(test)] +mod hardening_tests { + use super::*; + + // ── replication listener gate ───────────────────────────────────────────── + + #[test] + fn the_replication_port_stays_closed_unless_asked_for() { + // The default. Before this gate existed the listener bound + // 0.0.0.0:6381 on every node and, with no password, served the entire + // keyspace to anyone who connected — so an operator who set + // RECACHED_PASSWORD was protecting nothing. + assert_eq!(resolve_repl_listen(None, "0.0.0.0", None), Ok(false)); + assert_eq!(resolve_repl_listen(None, "0.0.0.0", Some("pw")), Ok(false)); + // An empty value is "unset", not "true". + assert_eq!( + resolve_repl_listen(Some(String::new()), "0.0.0.0", None), + Ok(false) + ); + assert_eq!( + resolve_repl_listen(Some(" ".to_string()), "0.0.0.0", None), + Ok(false) + ); + } + + #[test] + fn enabling_it_on_a_public_interface_without_a_password_refuses_to_start() { + let err = resolve_repl_listen(Some("1".into()), "0.0.0.0", None) + .expect_err("public + no password must not be allowed"); + // The message has to name both variables — an operator reading a log + // line needs to know what to set, not merely that something is wrong. + assert!(err.contains("RECACHED_REPL_PASSWORD"), "{err}"); + assert!(err.contains("RECACHED_REPL_ENABLE"), "{err}"); + assert!(err.contains("0.0.0.0"), "{err}"); + + // A specific LAN address is just as reachable as 0.0.0.0. + assert!(resolve_repl_listen(Some("1".into()), "10.0.1.5", None).is_err()); + // So is a hostname we cannot resolve to a loopback address: the + // conservative reading is the one that demands a password. + assert!(resolve_repl_listen(Some("1".into()), "cache.internal", None).is_err()); + // An empty password is not a password. + assert!(resolve_repl_listen(Some("1".into()), "0.0.0.0", Some("")).is_err()); + } + + #[test] + fn enabling_it_is_allowed_on_loopback_or_with_a_password() { + // Loopback without a password is a development setup, not an exposure. + assert_eq!( + resolve_repl_listen(Some("1".into()), "127.0.0.1", None), + Ok(true) + ); + assert_eq!( + resolve_repl_listen(Some("yes".into()), "::1", None), + Ok(true) + ); + assert_eq!( + resolve_repl_listen(Some("on".into()), "localhost", None), + Ok(true) + ); + // Public is fine once authenticated — this is the multi-tier + // replication path, which must keep working. + assert_eq!( + resolve_repl_listen(Some("true".into()), "0.0.0.0", Some("pw")), + Ok(true) + ); + assert_eq!( + resolve_repl_listen(Some("1".into()), "10.0.1.5", Some("pw")), + Ok(true) + ); + } + + #[test] + fn an_ambiguous_enable_value_refuses_to_start() { + // Treating `please` as false would leave an operator believing + // replication was on; treating it as true would open a port nobody + // asked for. Neither is acceptable for a variable gating a boundary. + let err = resolve_repl_listen(Some("please".into()), "127.0.0.1", None).unwrap_err(); + assert!(err.contains("RECACHED_REPL_ENABLE"), "{err}"); + assert!(err.contains("not a boolean"), "{err}"); + } + + #[test] + fn boolean_env_values_cover_the_conventional_spellings() { + for yes in ["1", "true", "TRUE", "yes", "On", " on "] { + assert_eq!(parse_env_bool("V", yes), Ok(true), "{yes:?}"); + } + for no in ["0", "false", "FALSE", "no", "Off", " off "] { + assert_eq!(parse_env_bool("V", no), Ok(false), "{no:?}"); + } + assert!(parse_env_bool("V", "maybe").is_err()); + } + + #[test] + fn loopback_detection_treats_unparseable_hosts_as_public() { + assert!(bind_is_loopback("127.0.0.1")); + assert!(bind_is_loopback("127.0.0.53")); + assert!(bind_is_loopback("::1")); + assert!(bind_is_loopback("localhost")); + assert!(bind_is_loopback("LOCALHOST")); + assert!(!bind_is_loopback("0.0.0.0")); + assert!(!bind_is_loopback("10.0.1.5")); + assert!(!bind_is_loopback("::")); + assert!(!bind_is_loopback("cache.internal")); + assert!(!bind_is_loopback("")); + // An IPv6 bind address must be written bracketed for the listeners to + // format it correctly, so the brackets have to be tolerated here too — + // otherwise `[::1]` is misread as public and demands a password. + assert!(bind_is_loopback("[::1]")); + assert!(!bind_is_loopback("[::]")); + assert!(!bind_is_loopback("[fd00::5]")); + } + + // ── replication auth: throttle and handshake ───────────────────────────── + + #[test] + fn repeated_bad_replication_passwords_block_the_peer() { + // The RESP port drops a connection after five guesses, but the + // replication handshake is one-shot: reconnecting used to reset the + // count, so the port offered unlimited guesses at a secret that yields + // the whole keyspace. The throttle is keyed by address for that reason. + let throttle = ReplAuthThrottle::new(); + let ip = IpAddr::from([203, 0, 113, 7]); + assert!(!throttle.is_blocked(ip)); + for _ in 0..MAX_AUTH_FAILURES { + assert!(!throttle.is_blocked(ip), "must not block before the cap"); + throttle.record_failure(ip); + } + assert!(throttle.is_blocked(ip), "cap reached, peer must be refused"); + + // Other peers are unaffected — one attacker must not lock out a fleet. + assert!(!throttle.is_blocked(IpAddr::from([203, 0, 113, 8]))); + + // A successful handshake clears the record. + throttle.record_success(ip); + assert!(!throttle.is_blocked(ip)); + } + + #[test] + fn the_throttle_does_not_grow_without_bound() { + // A spray from many source addresses must not be a memory-growth + // vector; the map sweeps once it crosses the threshold. + let throttle = ReplAuthThrottle::new(); + for i in 0..(REPL_AUTH_SWEEP_THRESHOLD + 64) { + let ip = IpAddr::from([ + 10, + ((i >> 16) & 0xff) as u8, + ((i >> 8) & 0xff) as u8, + (i & 0xff) as u8, + ]); + throttle.record_failure(ip); + } + let len = throttle.failures.lock().unwrap().len(); + // Entries are all fresh so none are swept, but the sweep must have run + // without panicking and the map must stay proportional to the input + // rather than duplicating it. + assert!(len <= REPL_AUTH_SWEEP_THRESHOLD + 64, "{len}"); + } + + #[tokio::test] + async fn the_auth_line_is_read_to_its_terminator_not_to_the_password_length() { + // Reading exactly `password.len() + 1` bytes made the number of bytes + // the server waited for *be* the password length, recoverable by + // drip-feeding one byte at a time. + let (mut client, mut server) = tokio::io::duplex(256); + client.write_all(b"hunter2\n").await.unwrap(); + let line = read_repl_auth_line(&mut server).await.unwrap(); + assert_eq!(line, b"hunter2"); + } + + #[tokio::test] + async fn an_auth_line_without_a_terminator_is_refused_at_the_cap() { + let (mut client, mut server) = tokio::io::duplex(4096); + let flood = vec![b'x'; MAX_REPL_AUTH_LINE + 16]; + // Write concurrently: the reader gives up mid-stream, so the writer + // must not block on a full pipe. + tokio::spawn(async move { + let _ = client.write_all(&flood).await; + }); + let err = read_repl_auth_line(&mut server) + .await + .expect_err("an unterminated line must not be read forever"); + assert_eq!(err.kind(), ErrorKind::InvalidData); + } + + #[tokio::test] + async fn a_short_auth_line_still_compares_unequal() { + // The comparison is constant-time and length-checked, so a truncated + // guess fails rather than matching a prefix. + let (mut client, mut server) = tokio::io::duplex(256); + client.write_all(b"hunt\n").await.unwrap(); + let line = read_repl_auth_line(&mut server).await.unwrap(); + assert!(!ct_eq_bytes(&line, b"hunter2")); + } + + // ── WebSocket origin allowlist ────────────────────────────────────────── + + #[test] + fn an_unset_origin_allowlist_permits_everything() { + // Matches how an unset RECACHED_PASSWORD behaves. The project ships + // insecure-by-default deliberately and says so; what it must not do is + // ship a *silent* default, hence the startup warning. + assert!(origin_allowed(None, Some("https://evil.example"))); + assert!(origin_allowed(None, None)); + } + + #[test] + fn a_foreign_origin_is_refused_when_the_allowlist_is_set() { + // The finding this closes: browsers apply neither CORS nor a preflight + // to WebSockets, so without this check any page a user visits could + // open a socket to ws://localhost:6380 and read or write every key. + let allow = vec!["https://app.example.com".to_string()]; + assert!(origin_allowed( + Some(&allow), + Some("https://app.example.com") + )); + assert!(!origin_allowed(Some(&allow), Some("https://evil.example"))); + // A different scheme or port is a different origin. + assert!(!origin_allowed( + Some(&allow), + Some("http://app.example.com") + )); + assert!(!origin_allowed( + Some(&allow), + Some("https://app.example.com:8443") + )); + // Substring matching would be a hole: `app.example.com.evil.test` + // contains an allowlisted origin as a prefix. + assert!(!origin_allowed( + Some(&allow), + Some("https://app.example.com.evil.test") + )); + } + + #[test] + fn an_absent_origin_is_permitted_because_only_browsers_send_one() { + // A native client omits the header and an attacker with a socket can + // forge it, so refusing here would break legitimate clients while + // stopping nobody. The control exists to separate "the app I deployed" + // from "another page in the same browser". + let allow = vec!["https://app.example.com".to_string()]; + assert!(origin_allowed(Some(&allow), None)); + } + + #[test] + fn origin_comparison_ignores_case_and_a_trailing_slash() { + let allow = parse_allowed_origins("https://App.Example.com/").unwrap(); + assert!(origin_allowed( + Some(&allow), + Some("https://app.example.com") + )); + assert!(origin_allowed( + Some(&allow), + Some("HTTPS://APP.EXAMPLE.COM/") + )); + } + + #[test] + fn the_origin_allowlist_parses_a_list_and_admits_null() { + let list = parse_allowed_origins( + "https://app.example.com, http://localhost:3000 ,https://admin.example.com:8443", + ) + .unwrap(); + assert_eq!( + list, + vec![ + "https://app.example.com", + "http://localhost:3000", + "https://admin.example.com:8443", + ] + ); + // Sandboxed iframes and file:// documents send the literal `null`. + assert_eq!(parse_allowed_origins("null").unwrap(), vec!["null"]); + assert!(origin_allowed( + Some(&parse_allowed_origins("null").unwrap()), + Some("null") + )); + } + + #[test] + fn the_origin_allowlist_rejects_entries_that_could_never_match() { + // Each of these would parse into something a browser never sends, so + // the allowlist would silently reject every connection. Failing at + // startup is the only way an operator finds out. + for bad in [ + "app.example.com", + "https://app.example.com/dashboard", + "://nohost", + "https://", + ] { + assert!( + parse_allowed_origins(bad).is_err(), + "{bad:?} should be rejected" + ); + } + // Set-but-empty would reject every browser; unset is how you allow all. + let err = parse_allowed_origins(" , ").unwrap_err(); + assert!(err.contains("Unset it"), "{err}"); + } + + // ── handshake deadline ────────────────────────────────────────────────── + + #[tokio::test] + async fn a_stalled_websocket_handshake_gives_up_and_releases_the_socket() { + // The connection permit is acquired *before* the handshake runs, so + // without this deadline `RECACHED_MAX_CONNECTIONS` sockets that connect + // and then say nothing — costing an attacker nothing — hold every slot + // indefinitely and the server stops accepting real clients. + let (_client, server) = tokio::io::duplex(1024); + let start = std::time::Instant::now(); + let out = ws_handshake(server, None, Duration::from_millis(150), 1).await; + assert!(out.is_none(), "a silent peer must not produce a stream"); + assert!( + start.elapsed() < Duration::from_secs(2), + "gave up after {:?} — the deadline did not apply", + start.elapsed() + ); + } + + #[test] + fn the_handshake_deadline_has_a_documented_default() { + assert_eq!(DEFAULT_HANDSHAKE_TIMEOUT_SECS, 10); + } + + // ── persistence file permissions ──────────────────────────────────────── + + #[tokio::test] + #[cfg(unix)] + async fn snapshot_and_sidecar_files_are_not_world_readable() { + use std::os::unix::fs::PermissionsExt; + + let dir = std::env::temp_dir().join(format!("recached_perm_{}", std::process::id())); + tokio::fs::create_dir_all(&dir).await.unwrap(); + let path = dir.join("perm-test.rdb"); + + write_private(&path, b"payload").await.unwrap(); + let mode = tokio::fs::metadata(&path) + .await + .unwrap() + .permissions() + .mode(); + assert_eq!( + mode & 0o777, + 0o600, + "snapshots are plaintext dumps of the keyspace; 0644 lets any local user read the cache" + ); + + // A file left behind 0644 by an earlier version must be tightened on + // the next write, not keep its old mode forever. + tokio::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o644)) + .await + .unwrap(); + write_private(&path, b"payload2").await.unwrap(); + let mode = tokio::fs::metadata(&path) + .await + .unwrap() + .permissions() + .mode(); + assert_eq!(mode & 0o777, 0o600, "an existing loose mode must be fixed"); + assert_eq!(tokio::fs::read(&path).await.unwrap(), b"payload2"); + + let _ = tokio::fs::remove_dir_all(&dir).await; + } + + #[tokio::test] + #[cfg(unix)] + async fn the_aof_is_not_world_readable() { + use std::os::unix::fs::PermissionsExt; + + let dir = std::env::temp_dir().join(format!("recached_aofperm_{}", std::process::id())); + tokio::fs::create_dir_all(&dir).await.unwrap(); + let path = dir.join("perm-test.aof"); + + // Pre-create it loose, as an upgrade from an earlier version would. + tokio::fs::write(&path, b"").await.unwrap(); + tokio::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o644)) + .await + .unwrap(); + + let writer = AofWriter::open(path.clone(), AofSync::No).await.unwrap(); + writer.append(b"*1\r\n$4\r\nPING\r\n").await; + let mode = tokio::fs::metadata(&path) + .await + .unwrap() + .permissions() + .mode(); + assert_eq!(mode & 0o777, 0o600); + + let _ = tokio::fs::remove_dir_all(&dir).await; + } + + #[test] + fn temp_files_do_not_collide_between_processes() { + // A fixed `.tmp` name meant two servers sharing a data directory would + // clobber each other's half-written snapshot. + let a = temp_sibling(std::path::Path::new("/data/recached.rdb"), "snap"); + assert!( + a.to_string_lossy() + .contains(&std::process::id().to_string()), + "{a:?}" + ); + assert!(a.to_string_lossy().ends_with(".tmp"), "{a:?}"); + assert_ne!( + a, + temp_sibling(std::path::Path::new("/data/recached.rdb"), "dedup") + ); + } +} + +/// A command that cannot be queued must poison the whole transaction. +/// +/// Redis refuses the command at queue time and makes `EXEC` reply `EXECABORT`, +/// having run nothing. Recached used to queue an unrecognised verb happily — +/// it parses to [`Command::Unknown`] — and only error while executing, so every +/// *other* command in the transaction was applied. On a server that implements +/// a deliberate subset of Redis that is a live hazard rather than a corner case: +/// `MULTI; ZPOPMIN q; LPUSH processing x; EXEC` pushed onto `processing` +/// without ever popping `q`, and MULTI is precisely the construct a caller +/// reaches for to stop exactly that. +#[cfg(test)] +mod transaction_abort_tests { + use super::*; + use super::{RespClient, spawn_server}; + + fn is_execabort(v: &Value) -> bool { + matches!(v, Value::Error(e) if e.starts_with("EXECABORT")) + } + + #[tokio::test] + async fn an_unknown_command_is_refused_when_queued_not_when_executed() { + let srv = spawn_server().await; + let mut c = RespClient::connect(srv.tcp_addr).await; + + assert_eq!(c.cmd(&["MULTI"]).await, Value::SimpleString("OK".into())); + let reply = c.cmd(&["NOSUCHCMD", "x"]).await; + assert!( + matches!(&reply, Value::Error(e) if e.contains("unknown command") && e.contains("NOSUCHCMD")), + "an unknown verb must be refused at queue time, got {reply:?}" + ); + assert_ne!( + reply, + Value::SimpleString("QUEUED".into()), + "a command that will never run must not be acknowledged as QUEUED" + ); + } + + #[tokio::test] + async fn exec_after_a_failed_queue_runs_nothing() { + let srv = spawn_server().await; + let mut c = RespClient::connect(srv.tcp_addr).await; + + c.cmd(&["MULTI"]).await; + c.cmd(&["NOSUCHCMD", "x"]).await; // refused, poisons the transaction + assert_eq!( + c.cmd(&["SET", "survivor", "yes"]).await, + Value::SimpleString("QUEUED".into()) + ); + let exec = c.cmd(&["EXEC"]).await; + assert!(is_execabort(&exec), "expected EXECABORT, got {exec:?}"); + + // The point of the whole fix: the sibling write must not have landed. + assert_eq!( + c.cmd(&["GET", "survivor"]).await, + Value::BulkString(None), + "a transaction that was refused still applied one of its commands" + ); + } + + #[tokio::test] + async fn a_malformed_command_also_aborts_the_transaction() { + // Wrong arity fails in `Command::from_value`, a different rejection path + // from `Command::Unknown` — both must poison the transaction. + let srv = spawn_server().await; + let mut c = RespClient::connect(srv.tcp_addr).await; + + c.cmd(&["MULTI"]).await; + let bad = c.cmd(&["INCR"]).await; // missing key + assert!(matches!(bad, Value::Error(_)), "got {bad:?}"); + c.cmd(&["SET", "arity:survivor", "yes"]).await; + assert!(is_execabort(&c.cmd(&["EXEC"]).await)); + assert_eq!( + c.cmd(&["GET", "arity:survivor"]).await, + Value::BulkString(None) + ); + } + + #[tokio::test] + async fn a_command_not_allowed_in_a_transaction_aborts_it() { + let srv = spawn_server().await; + let mut c = RespClient::connect(srv.tcp_addr).await; + + c.cmd(&["MULTI"]).await; + let refused = c.cmd(&["SUBSCRIBE", "ch"]).await; + assert!(matches!(refused, Value::Error(_)), "got {refused:?}"); + c.cmd(&["SET", "sub:survivor", "yes"]).await; + assert!(is_execabort(&c.cmd(&["EXEC"]).await)); + assert_eq!( + c.cmd(&["GET", "sub:survivor"]).await, + Value::BulkString(None) + ); + } + + #[tokio::test] + async fn a_clean_transaction_still_executes() { + // Regression guard: the abort path must not swallow ordinary work. + let srv = spawn_server().await; + let mut c = RespClient::connect(srv.tcp_addr).await; + + c.cmd(&["MULTI"]).await; + c.cmd(&["SET", "clean", "v"]).await; + c.cmd(&["INCR", "clean:n"]).await; + let exec = c.cmd(&["EXEC"]).await; + assert_eq!( + exec, + Value::Array(Some(vec![ + Value::SimpleString("OK".into()), + Value::Integer(1), + ])), + "a transaction with no queue errors must run in full" + ); + assert_eq!( + c.cmd(&["GET", "clean"]).await, + Value::BulkString(Some(b"v".to_vec())) + ); + } + + #[tokio::test] + async fn discard_clears_the_abort_state() { + // The flag is per-transaction, not per-connection: a poisoned + // transaction must not wedge every later one on the same socket. + let srv = spawn_server().await; + let mut c = RespClient::connect(srv.tcp_addr).await; + + c.cmd(&["MULTI"]).await; + c.cmd(&["NOSUCHCMD"]).await; + assert_eq!(c.cmd(&["DISCARD"]).await, Value::SimpleString("OK".into())); + + c.cmd(&["MULTI"]).await; + c.cmd(&["SET", "after:discard", "v"]).await; + assert_eq!( + c.cmd(&["EXEC"]).await, + Value::Array(Some(vec![Value::SimpleString("OK".into())])) + ); + assert_eq!( + c.cmd(&["GET", "after:discard"]).await, + Value::BulkString(Some(b"v".to_vec())) + ); + } + + #[tokio::test] + async fn a_new_multi_clears_the_abort_state_left_by_an_aborted_one() { + let srv = spawn_server().await; + let mut c = RespClient::connect(srv.tcp_addr).await; + + c.cmd(&["MULTI"]).await; + c.cmd(&["NOSUCHCMD"]).await; + assert!(is_execabort(&c.cmd(&["EXEC"]).await)); + + // Next transaction on the same connection starts clean. + c.cmd(&["MULTI"]).await; + c.cmd(&["SET", "after:abort", "v"]).await; + assert_eq!( + c.cmd(&["EXEC"]).await, + Value::Array(Some(vec![Value::SimpleString("OK".into())])), + "the abort flag leaked into the next transaction" + ); + assert_eq!( + c.cmd(&["GET", "after:abort"]).await, + Value::BulkString(Some(b"v".to_vec())) + ); + } + + #[tokio::test] + async fn a_watch_abort_stays_distinct_from_a_queue_abort() { + // Two different failures with two different replies: a CAS conflict is + // a nil array (retry me), a refused command is EXECABORT (fix your + // request). Collapsing them would make retry loops spin forever. + let srv = spawn_server().await; + let mut c = RespClient::connect(srv.tcp_addr).await; + let mut other = RespClient::connect(srv.tcp_addr).await; + + c.cmd(&["SET", "w", "1"]).await; + c.cmd(&["WATCH", "w"]).await; + c.cmd(&["MULTI"]).await; + c.cmd(&["SET", "w", "99"]).await; + other.cmd(&["SET", "w", "42"]).await; // invalidates the watch + tokio::time::sleep(tokio::time::Duration::from_millis(50)).await; + + let exec = c.cmd(&["EXEC"]).await; + assert_eq!(exec, Value::Array(None), "CAS conflict must reply nil"); + assert!(!is_execabort(&exec)); + assert_eq!( + c.cmd(&["GET", "w"]).await, + Value::BulkString(Some(b"42".to_vec())) + ); + } + + #[test] + fn queue_time_rejection_names_the_command_and_matches_the_store() { + // One mistake should read the same inside and outside a transaction. + let cmd = Command::Unknown("ZPOPMIN".into()); + let refusal = queue_time_rejection(&cmd).expect("unknown verbs are refused"); + let text = String::from_utf8_lossy(&refusal).into_owned(); + assert_eq!(text, "-ERR unknown command 'ZPOPMIN'\r\n"); + + let store = core_engine::store::KeyValueStore::new(); + let direct = store.execute(cmd); + assert_eq!(Value::parse(&refusal).unwrap().0, direct); + } + + #[test] + fn a_queueable_command_is_not_rejected() { + assert!(queue_time_rejection(&Command::Get("k".into())).is_none()); + assert!( + queue_time_rejection(&Command::Set( + "k".into(), + "v".into(), + core_engine::cmd::SetOptions::default() + )) + .is_none() + ); + } + + #[test] + fn the_execabort_wire_text_matches_redis() { + // Clients match on this prefix to tell a refused transaction from one + // whose commands merely returned errors. + assert_eq!( + String::from_utf8_lossy(EXECABORT), + "-EXECABORT Transaction discarded because of previous errors.\r\n" + ); + } +} + +// ── Counter TTL propagation ─────────────────────────────────────────────────── + +/// `PUBLISH` is queueable, and actually delivers when `EXEC` runs it. +/// +/// Redis allows `PUBLISH` inside a transaction — announcing a change atomically +/// with the write that caused it is the ordinary reason to reach for MULTI at +/// all — but Recached refused it alongside `SUBSCRIBE` and `WATCH`. Simply +/// allowing it to queue would have been worse than the refusal: delivery lives +/// in the connection loop, and `store.execute(Publish)` is a stub that answers +/// 0 and sends nothing, so the message would have been swallowed silently. The +/// EXEC loop therefore dispatches `Publish` to the hub itself. +#[cfg(test)] +mod publish_in_multi_tests { + use super::*; + use super::{RespClient, spawn_server}; + + #[tokio::test] + async fn publish_can_be_queued_inside_a_transaction() { + let srv = spawn_server().await; + let mut c = RespClient::connect(srv.tcp_addr).await; + + assert_eq!(c.cmd(&["MULTI"]).await, Value::SimpleString("OK".into())); + assert_eq!( + c.cmd(&["PUBLISH", "events", "hi"]).await, + Value::SimpleString("QUEUED".into()), + "Redis allows PUBLISH inside MULTI" + ); + // No subscribers, so the reply is a delivery count of zero. + assert_eq!( + c.cmd(&["EXEC"]).await, + Value::Array(Some(vec![Value::Integer(0)])) + ); + } + + #[tokio::test] + async fn a_queued_publish_actually_reaches_a_subscriber() { + // The part that a delivery count alone would not prove: the stub in the + // store answers 0 and sends nothing, so a wrongly-wired EXEC would look + // fine on the publisher's side while the message vanished. + let srv = spawn_server().await; + let mut sub = RespClient::connect(srv.tcp_addr).await; + let mut pubr = RespClient::connect(srv.tcp_addr).await; + + sub.cmd(&["SUBSCRIBE", "events"]).await; // subscribe ack + tokio::time::sleep(tokio::time::Duration::from_millis(50)).await; + + pubr.cmd(&["MULTI"]).await; + pubr.cmd(&["SET", "order:1", "paid"]).await; + pubr.cmd(&["PUBLISH", "events", "order:1 paid"]).await; + let exec = pubr.cmd(&["EXEC"]).await; + assert_eq!( + exec, + Value::Array(Some(vec![ + Value::SimpleString("OK".into()), + Value::Integer(1), // one subscriber received it + ])), + "EXEC must report the real delivery count, not the store's stub" + ); + + // And the subscriber genuinely has the message. + let msg = sub.cmd(&[]).await; + assert_eq!( + msg, + Value::Array(Some(vec![ + Value::BulkString(Some(b"message".to_vec())), + Value::BulkString(Some(b"events".to_vec())), + Value::BulkString(Some(b"order:1 paid".to_vec())), + ])), + "the queued PUBLISH was swallowed" + ); + + // The write in the same transaction landed too. + assert_eq!( + pubr.cmd(&["GET", "order:1"]).await, + Value::BulkString(Some(b"paid".to_vec())) + ); + } + + #[tokio::test] + async fn a_discarded_publish_is_never_delivered() { + // Queued means pending, not sent: DISCARD must drop the message. + let srv = spawn_server().await; + let mut sub = RespClient::connect(srv.tcp_addr).await; + let mut pubr = RespClient::connect(srv.tcp_addr).await; + + sub.cmd(&["SUBSCRIBE", "events"]).await; + tokio::time::sleep(tokio::time::Duration::from_millis(50)).await; + + pubr.cmd(&["MULTI"]).await; + pubr.cmd(&["PUBLISH", "events", "never"]).await; + assert_eq!( + pubr.cmd(&["DISCARD"]).await, + Value::SimpleString("OK".into()) + ); + + // Publish something real; the subscriber must see *that* first, which + // it could not if the discarded message had already been sent. + pubr.cmd(&["PUBLISH", "events", "real"]).await; + let msg = sub.cmd(&[]).await; + assert_eq!( + msg, + Value::Array(Some(vec![ + Value::BulkString(Some(b"message".to_vec())), + Value::BulkString(Some(b"events".to_vec())), + Value::BulkString(Some(b"real".to_vec())), + ])), + "a DISCARDed PUBLISH was delivered anyway" + ); + } + + #[tokio::test] + async fn subscribe_and_watch_are_still_refused_inside_a_transaction() { + // Only PUBLISH was unqueueable-by-mistake; the others are correct. + let srv = spawn_server().await; + let mut c = RespClient::connect(srv.tcp_addr).await; + + c.cmd(&["MULTI"]).await; + for args in [ + vec!["SUBSCRIBE", "ch"], + vec!["PSUBSCRIBE", "ch:*"], + vec!["WATCH", "k"], + ] { + let reply = c.cmd(&args).await; + assert!( + matches!(&reply, Value::Error(e) if e.contains("not allowed inside a transaction")), + "{args:?} should still be refused, got {reply:?}" + ); + } + } +} + +// ── Metrics port ────────────────────────────────────────────────────────────── diff --git a/server-native/src/tls.rs b/server-native/src/tls.rs new file mode 100644 index 0000000..a4f40e0 --- /dev/null +++ b/server-native/src/tls.rs @@ -0,0 +1,277 @@ +//! TLS material: certificate/key loading for the listener and for outbound +//! replication. + +use crate::*; + +// PEM parsing comes from rustls-pki-types, the crate rustls itself uses. +// `rustls-pemfile` was deprecated in favour of it (RUSTSEC-2025-0134), and an +// unmaintained dependency is a poor thing to have sitting in the TLS path. +pub(crate) fn load_certs(path: &str) -> std::io::Result>> { + CertificateDer::pem_file_iter(path) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string()))? + .map(|r| r.map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string()))) + .collect() +} + +pub(crate) fn load_private_key(path: &str) -> std::io::Result> { + PrivateKeyDer::from_pem_file(path) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e.to_string())) +} + +/// Decides what a (cert, key) environment pair means, without touching the +/// filesystem — split out from `load_tls_acceptor` so the security-relevant +/// rule is unit-testable. +/// +/// Setting only one half is treated as a fatal misconfiguration rather than a +/// fallback to plaintext: an operator who set `RECACHED_TLS_CERT` intends TLS, +/// and silently serving unencrypted traffic on both ports because the key +/// variable was misspelled is a failure they would not detect until traffic had +/// already been exposed. +pub(crate) fn resolve_tls_paths( + cert: Option, + key: Option, +) -> Result, String> { + match (cert, key) { + (None, None) => Ok(None), + (Some(c), Some(k)) => Ok(Some((c, k))), + (Some(_), None) => Err( + "RECACHED_TLS_CERT is set but RECACHED_TLS_KEY is not — refusing to start rather than \ + silently serving plaintext. Set both, or neither." + .to_string(), + ), + (None, Some(_)) => Err( + "RECACHED_TLS_KEY is set but RECACHED_TLS_CERT is not — refusing to start rather than \ + silently serving plaintext. Set both, or neither." + .to_string(), + ), + } +} + +/// The name a replica verifies the primary's certificate against. +/// +/// Defaults to the host portion of `RECACHED_REPLICAOF`, which is what an +/// operator means by "connect to this primary". It is overridable because the +/// address is frequently an IP while the certificate names a host: a cert issued +/// for `primary.internal` does not validate against `10.0.1.5` unless it also +/// carries that IP as a SAN, and pointing at the IP is the common deployment. +pub(crate) fn repl_tls_servername(primary_addr: &str, override_name: Option) -> String { + if let Some(name) = override_name.map(|n| n.trim().to_string()) + && !name.is_empty() + { + return name; + } + // `host:port`, or a bare host. An IPv6 literal is bracketed, so splitting on + // the last colon would cut inside the address. + match primary_addr.rsplit_once(':') { + Some((host, _)) if !host.is_empty() && !host.contains(':') => host.to_string(), + _ => primary_addr + .trim_start_matches('[') + .split(']') + .next() + .unwrap_or(primary_addr) + .to_string(), + } +} + +/// Build the TLS connector a replica uses to reach its primary. +/// +/// The trust anchor is an explicit file rather than the system root store, and +/// that is deliberate. Replication is a link between two machines the same +/// operator runs, so the right model is pinning the certificate (or the private +/// CA that issued it) — not trusting every public CA on earth to vouch for a +/// host that streams the entire keyspace. Pointing this at a system bundle still +/// works if the primary genuinely uses a publicly-issued certificate. +pub(crate) fn load_repl_tls_connector(ca_path: &str) -> Result { + let certs = + load_certs(ca_path).map_err(|e| format!("RECACHED_REPL_TLS_CA '{ca_path}': {e}"))?; + if certs.is_empty() { + return Err(format!( + "RECACHED_REPL_TLS_CA '{ca_path}' contains no certificates — replication TLS would \ + trust nothing and every connection would fail." + )); + } + let mut roots = RootCertStore::empty(); + for cert in certs { + roots + .add(cert) + .map_err(|e| format!("RECACHED_REPL_TLS_CA '{ca_path}': {e}"))?; + } + let config = ClientConfig::builder() + .with_root_certificates(roots) + .with_no_client_auth(); + Ok(TlsConnector::from(Arc::new(config))) +} + +/// Returns a `TlsAcceptor` when both `RECACHED_TLS_CERT` and `RECACHED_TLS_KEY` +/// are set, `None` when neither is. Exits if exactly one is set. +pub(crate) fn load_tls_acceptor() -> Option { + let (cert_path, key_path) = match resolve_tls_paths( + std::env::var("RECACHED_TLS_CERT").ok(), + std::env::var("RECACHED_TLS_KEY").ok(), + ) { + Ok(None) => return None, + Ok(Some(pair)) => pair, + Err(msg) => { + error!("{msg}"); + std::process::exit(1); + } + }; + + let cert_coll = load_certs(&cert_path).unwrap_or_else(|e| panic!("TLS cert {cert_path}: {e}")); + let key = load_private_key(&key_path).unwrap_or_else(|e| panic!("TLS key {key_path}: {e}")); + + let config = ServerConfig::builder() + .with_no_client_auth() + .with_single_cert(cert_coll, key) + .expect("invalid TLS configuration"); + + Some(TlsAcceptor::from(Arc::new(config))) +} + +// ── tunables ──────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tls_loading_tests { + use super::*; + + // A self-signed cert and its key, generated once with: + // openssl req -x509 -newkey rsa:2048 -keyout k -out c -days 3650 -nodes \ + // -subj "/CN=recached-test" + // Embedded rather than generated at test time so the test needs no openssl + // on the runner and cannot fail for reasons unrelated to parsing. + const TEST_CERT: &str = "-----BEGIN CERTIFICATE-----\nMIIDETCCAfmgAwIBAgIUDpGtGZ5z4j/X0RMdVgiZt5TyukwwDQYJKoZIhvcNAQEL\nBQAwGDEWMBQGA1UEAwwNcmVjYWNoZWQtdGVzdDAeFw0yNjA3MTkxNDUyMDVaFw0z\nNjA3MTYxNDUyMDVaMBgxFjAUBgNVBAMMDXJlY2FjaGVkLXRlc3QwggEiMA0GCSqG\nSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDdi5zyxNocCEi6elQKsS0onYh9aOMW5Hjz\n7zAcWa6EPp1g4Zz1tLF2Nk92CBG/iWzF5OckDChuIYjM+MTRws5UOSXwwkbLplKR\nSMGEst1mP3rZPGHq57w52OmxO599kBR4BpeWhFMC4w5xGEO9Gp4P+QdCIYaUEBxz\nLeEyCwapimzamKRYKO0VoZWzF0bLhYUHxc9FD2QMbaPUmRZZGdcttg/0Gq4U/P5N\n6jhWo+ekIKu1kpLSAZPiHtYNAzGu1sk0lTPyVxdmmwqPueV9MLUgVIpDWA+QL80I\nXIjTfaQAOl4k31AeC+yglCyhB/yl/0ROQUAXGgozsFJnpxujLGMPAgMBAAGjUzBR\nMB0GA1UdDgQWBBSWbJJErt4zE9+u8lbBnAPXaRSI0TAfBgNVHSMEGDAWgBSWbJJE\nrt4zE9+u8lbBnAPXaRSI0TAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUA\nA4IBAQBGslzIW0Q46r7eQGK22fTEfNReSy4f7PZPGn/BZbj499LKSfRP1z8A3bbF\n2CdQKswbhVUbHfLUoaRwRfmJWhR/I/UxNkUfVlQ/jQBaUvg2ZCy1l/3kRM6N1t5K\ntkwg+dzai/6LwT7RHmbl8Dx32on3+x9vJMYtoxeBk4nfHZTQMIOd3zsaXp/+RWUY\nzuIWXX/rf862GerYhoHVCWzMcHMLnI/Mwzlm2tgVnfW1XpI/La3fxnTWYT4g4PIJ\nfXe3WrO9VyC1ZZ7PjE4Pq4unCRbJ2yZ5toybr4kcT4UGFrsXjnAsT+RyLY4By50D\nkaBPsvjq5ZvbiPBtEINXbmF3A7cq\n-----END CERTIFICATE-----"; + const TEST_KEY: &str = "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDdi5zyxNocCEi6\nelQKsS0onYh9aOMW5Hjz7zAcWa6EPp1g4Zz1tLF2Nk92CBG/iWzF5OckDChuIYjM\n+MTRws5UOSXwwkbLplKRSMGEst1mP3rZPGHq57w52OmxO599kBR4BpeWhFMC4w5x\nGEO9Gp4P+QdCIYaUEBxzLeEyCwapimzamKRYKO0VoZWzF0bLhYUHxc9FD2QMbaPU\nmRZZGdcttg/0Gq4U/P5N6jhWo+ekIKu1kpLSAZPiHtYNAzGu1sk0lTPyVxdmmwqP\nueV9MLUgVIpDWA+QL80IXIjTfaQAOl4k31AeC+yglCyhB/yl/0ROQUAXGgozsFJn\npxujLGMPAgMBAAECggEAQ4xj6ClZDx7/fcv6f+ARksalbQdj5gD3V/jfxGUbrrqg\npX9kqg3T5eUdSTGgp7Ow9I2cZANI+HtFCKn46LPq0QczqDqz9zfZCO8UAe+/TYOh\nY0bj3AmX/FNEvYMeV9xsQURRR9VEsiakqprpXGkXNGuLaQBr1g0rf3rHpMhz2ZEH\nw7QxoUfH3YL7fMIWwAvHanl6HwzE3TVh3felFGGGiqUaGg2Pll+/s5AiYnnZuq39\nW+t0RVH36rSFh037Su/ScCs44WZS+kGyqcuxyWLwvNwXEVXC3h42N62MHGgY8xQw\nP8wMSxGejEIr8IGpwhl86+oW+44nxarmrMBzMPRIcQKBgQD6VnpsIwxWV1zNC1LD\nbTVRQOoqJWDzxdXB47IB29ADHJnfBLbMr/6kIgIfuCu1Kf9wWTZGHwFUOKFhJDBC\ngLHpWFMWKSew4UmNyc0a16a9Pb7mdNyxnTY1oQucEbzS3pLMzda2dAdxYJE0Cuc6\nJ8Xp2Jo73LnRxY+NyEyl28frAwKBgQDijmtG0cDESNPMhxqJO/9KoRqRT3T+JqNf\noWaKlfSlQFaGecjdk9dNPZ1Aew4xI0v/C5YTwT6MUVDEXmoaSsa+S1atoDLNsojL\nuWqUno9mF6o3U23pi4vlEYh6c/V7Bd1VYde8ZQqVq0KxbCmYp1VE+DBeyNNT7Q5X\nN2lst0hEBQKBgQCXjds3tFA3xVQNXpmQboEk2+Pn+BEmA9NROoP91BGukJYnCjeQ\n28uRmnUmttzfJLncTmYpNYQcdNxebwY4fKk415wVgnzg/MMG7/EYGw566vKzmnQx\noze6Z/EbXzGth8nf643dj4kh/pBprWAnOQT8eYGGVC667Jvn/idJEjGJ+QKBgQCz\nGmgQio3cHr7huATwbO/7rbT1H12b9iu91DjeYoIPifddRDXZhaD1vTnt2dp0WjUg\nIaa5Y1HxV++D7ifvNSI9Gg4iIL1JBFVEyQZLC7bNvPOh3WDM+rbTlrLQK4/re81o\nTHtiwnZFsCh/XsTbm527coG6zQTUGln19SZw/cwxiQKBgA8dcEyBvPi6JgAqLy+5\n3Ev1uZEKkAeAQAkOV9jzqDN9NTi7GWOz3mtY2zopYjef7Wl0V4Qjkr7Jkxlx2wyn\nHboOuCEjComkRxn5vrHm6EBp0uTrdFIknLysxmQFgNamp9E8mX9p/q9rq7aZWzPu\nr+3jOYvwFyzAQ4j2tGzUm7Zd\n-----END PRIVATE KEY-----"; + + fn write(name: &str, body: &str) -> std::path::PathBuf { + let path = + std::env::temp_dir().join(format!("recached_test_{name}_{}", std::process::id())); + std::fs::write(&path, body).expect("write pem"); + path + } + + // ── replication TLS (2.2) ─────────────────────────────────────────────── + + #[test] + fn the_verified_servername_defaults_to_the_primarys_host() { + // What an operator means by "connect to this primary" is the host they + // named, so that is what the certificate is checked against. + assert_eq!( + repl_tls_servername("primary.internal:6381", None), + "primary.internal" + ); + assert_eq!(repl_tls_servername("10.0.1.5:6381", None), "10.0.1.5"); + assert_eq!( + repl_tls_servername("primary.internal", None), + "primary.internal" + ); + } + + #[test] + fn an_ipv6_primary_address_is_not_split_inside_the_address() { + // Splitting on the last colon would cut inside an IPv6 literal and + // produce a servername that could never validate. + assert_eq!(repl_tls_servername("[::1]:6381", None), "::1"); + assert_eq!(repl_tls_servername("[fd00::5]:6381", None), "fd00::5"); + } + + #[test] + fn the_servername_override_wins_and_ignores_blanks() { + // The common deployment points RECACHED_REPLICAOF at an IP while the + // certificate names a host, which cannot validate without an IP SAN. + assert_eq!( + repl_tls_servername("10.0.1.5:6381", Some("primary.internal".into())), + "primary.internal" + ); + assert_eq!( + repl_tls_servername("10.0.1.5:6381", Some(" primary.internal ".into())), + "primary.internal" + ); + // Empty or whitespace is "unset", not "verify against nothing". + assert_eq!( + repl_tls_servername("10.0.1.5:6381", Some(String::new())), + "10.0.1.5" + ); + assert_eq!( + repl_tls_servername("10.0.1.5:6381", Some(" ".into())), + "10.0.1.5" + ); + } + + #[test] + fn a_missing_or_unusable_replication_ca_is_a_startup_error() { + // Trusting nothing would fail every connection at runtime rather than at + // startup, which is much harder to diagnose from a replica's logs. + // `TlsConnector` is not Debug, so unwrap the error side by hand. + let missing = load_repl_tls_connector("/nonexistent/recached-ca.pem"); + let Err(err) = missing else { + panic!("a missing CA file must be refused"); + }; + assert!(err.contains("RECACHED_REPL_TLS_CA"), "{err}"); + + let junk = write("repl_ca_junk.pem", "not a certificate\n"); + let unusable = load_repl_tls_connector(junk.to_str().unwrap()); + let _ = std::fs::remove_file(&junk); + let Err(err) = unusable else { + panic!("a file with no certificates must be refused"); + }; + assert!(err.contains("RECACHED_REPL_TLS_CA"), "{err}"); + } + + #[test] + fn a_self_signed_certificate_works_as_the_replication_trust_anchor() { + // Pinning the primary's own certificate is the documented path, and the + // reason the trust anchor is an explicit file rather than the system root + // store: replication is a private link between two hosts one operator + // runs, so trusting every public CA to vouch for it would be backwards. + let ca = write("repl_ca_ok.pem", TEST_CERT); + assert!( + load_repl_tls_connector(ca.to_str().unwrap()).is_ok(), + "a valid self-signed PEM must build a connector" + ); + let _ = std::fs::remove_file(&ca); + } + + #[test] + fn a_pem_certificate_and_key_load() { + // PEM parsing moved from the deprecated rustls-pemfile to + // rustls-pki-types. These two functions had no coverage at all, so the + // swap would have been verified only by the code compiling. + let cert_path = write("tls_load.crt", TEST_CERT); + let key_path = write("tls_load.key", TEST_KEY); + + let certs = load_certs(cert_path.to_str().unwrap()).expect("cert must parse"); + assert_eq!(certs.len(), 1, "one certificate in the chain"); + assert!(!certs[0].as_ref().is_empty(), "DER body must be non-empty"); + + let key = load_private_key(key_path.to_str().unwrap()).expect("key must parse"); + assert!(!key.secret_der().is_empty(), "key DER must be non-empty"); + + let _ = std::fs::remove_file(&cert_path); + let _ = std::fs::remove_file(&key_path); + } + + #[test] + fn a_missing_file_is_an_error_not_a_panic() { + assert!(load_certs("/nonexistent/recached-test.crt").is_err()); + assert!(load_private_key("/nonexistent/recached-test.key").is_err()); + } + + #[test] + fn a_file_with_no_pem_content_is_rejected() { + // Pointing RECACHED_TLS_CERT at the wrong file must fail loudly rather + // than yielding an empty chain that rustls would later reject with a + // much less obvious error. + let junk = write("tls_junk.crt", "this is not a PEM file\n"); + assert!( + load_certs(junk.to_str().unwrap()) + .map(|c| c.is_empty()) + .unwrap_or(true), + "non-PEM input must not yield certificates" + ); + let junk_key = write("tls_junk.key", "still not PEM\n"); + assert!(load_private_key(junk_key.to_str().unwrap()).is_err()); + + let _ = std::fs::remove_file(&junk); + let _ = std::fs::remove_file(&junk_key); + } +} diff --git a/server-native/src/watch.rs b/server-native/src/watch.rs new file mode 100644 index 0000000..43537ec --- /dev/null +++ b/server-native/src/watch.rs @@ -0,0 +1,96 @@ +//! Watched keys: the registry behind WATCH's compare-and-set transactions and +//! the live-query (QSUB) subscriptions that share it. + +use crate::*; + +pub(crate) type WatchNotif = (String, Value); + +pub(crate) type WatchMap = HashMap)>>; + +/// Watched-key and live-query registry. `watched_keys` / `watched_patterns` +/// mirror the map lengths (updated by every writer while holding the lock) so +/// the per-write hot path can skip the mutexes entirely when nothing is +/// watched. +pub(crate) struct WatchHub { + /// Exact-key watchers (WATCH). + pub(crate) map: tokio::sync::Mutex, + pub(crate) watched_keys: AtomicUsize, + /// Glob-pattern subscribers (QSUB live queries), keyed by pattern. + pub(crate) patterns: tokio::sync::Mutex, + pub(crate) watched_patterns: AtomicUsize, +} + +impl WatchHub { + pub(crate) fn new() -> WatchRegistry { + Arc::new(WatchHub { + map: tokio::sync::Mutex::new(HashMap::new()), + watched_keys: AtomicUsize::new(0), + patterns: tokio::sync::Mutex::new(HashMap::new()), + watched_patterns: AtomicUsize::new(0), + }) + } + + pub(crate) fn is_empty(&self) -> bool { + self.watched_keys.load(Ordering::Relaxed) == 0 + && self.watched_patterns.load(Ordering::Relaxed) == 0 + } + + /// Call after mutating the key map, while still holding the lock. + pub(crate) fn sync_len(&self, map: &WatchMap) { + self.watched_keys.store(map.len(), Ordering::Relaxed); + } + + /// Call after mutating the pattern map, while still holding the lock. + pub(crate) fn sync_patterns_len(&self, map: &WatchMap) { + self.watched_patterns.store(map.len(), Ordering::Relaxed); + } +} + +pub(crate) type WatchRegistry = Arc; + +/// Drop all of `conn_id`'s live-query subscriptions. Called on QUNSUB (all +/// form) and on connection close. +pub(crate) async fn unregister_all_qsubs( + registry: &WatchRegistry, + conn_id: u64, + qsub_patterns: &mut HashSet, +) { + if qsub_patterns.is_empty() { + return; + } + let mut pats = registry.patterns.lock().await; + for p in qsub_patterns.drain() { + if let Some(subs) = pats.get_mut(&p) { + subs.retain(|(id, _)| *id != conn_id); + if subs.is_empty() { + pats.remove(&p); + } + } + } + registry.sync_patterns_len(&pats); +} + +/// Drop all of `conn_id`'s WATCH registrations and clear `watched_keys`. +/// Called at every transaction boundary (EXEC, DISCARD) and on connection close, +/// matching Redis semantics that WATCH state is flushed by EXEC/DISCARD. +pub(crate) async fn unregister_all_watches( + registry: &WatchRegistry, + conn_id: u64, + watched_keys: &mut HashSet, +) { + if watched_keys.is_empty() { + return; + } + let mut reg = registry.map.lock().await; + for key in watched_keys.drain() { + if let Some(subs) = reg.get_mut(&key) { + subs.retain(|(id, _)| *id != conn_id); + if subs.is_empty() { + reg.remove(&key); + } + } + } + registry.sync_len(®); +} + +// ── helpers ────────────────────────────────────────────────────────────────── diff --git a/wasm-edge/package.json b/wasm-edge/package.json index 93474cd..0fb325f 100644 --- a/wasm-edge/package.json +++ b/wasm-edge/package.json @@ -1,7 +1,7 @@ { "name": "recached-edge", "description": "Browser and edge WebAssembly client for Recached \u2014 zero-latency local cache with automatic server sync", - "version": "0.2.4", + "version": "0.2.5", "type": "module", "main": "sdk.js", "module": "sdk.js",