diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4942e6e..e5ee856 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -48,6 +48,16 @@ Prepare the cross-language serialization test data: cargo x prepare-testdata ``` +Generate deterministic snapshots from the Rust implementations: + +```shell +cargo x generate-snapshots +``` + +The command writes local files under `serialization/rust/snapshots`. These +generated files are ignored by Git, so the command can be rerun without +changing the working tree. + Test: ```shell diff --git a/Cargo.lock b/Cargo.lock index fd17711..849164c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -602,6 +602,14 @@ version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" +[[package]] +name = "snapshot_generator" +version = "0.0.0" +dependencies = [ + "clap", + "datasketches", +] + [[package]] name = "socks" version = "0.3.4" diff --git a/Cargo.toml b/Cargo.toml index 04f944e..981d4ef 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,7 +16,7 @@ # under the License. [workspace] -members = ["datasketches", "xtask"] +members = ["datasketches", "snapshot_generator", "xtask"] resolver = "3" [workspace.package] @@ -30,6 +30,7 @@ rust-version = "1.86.0" [workspace.dependencies] # Workspace dependencies datasketches = { path = "datasketches" } +snapshot_generator = { path = "snapshot_generator" } # Crates.io dependencies clap = { version = "4.6.5", features = ["derive"] } diff --git a/snapshot_generator/Cargo.toml b/snapshot_generator/Cargo.toml new file mode 100644 index 0000000..7a74fa4 --- /dev/null +++ b/snapshot_generator/Cargo.toml @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +name = "snapshot_generator" +publish = false + +edition.workspace = true +homepage.workspace = true +license.workspace = true +readme.workspace = true +repository.workspace = true +rust-version.workspace = true + +[dependencies] +clap = { workspace = true } +datasketches = { workspace = true, features = ["bloom", "countmin"] } + +[lints] +workspace = true diff --git a/snapshot_generator/src/main.rs b/snapshot_generator/src/main.rs new file mode 100644 index 0000000..d852157 --- /dev/null +++ b/snapshot_generator/src/main.rs @@ -0,0 +1,83 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::fs; +use std::io; +use std::path::PathBuf; + +use clap::Parser; +use datasketches::bloom::BloomFilterBuilder; +use datasketches::countmin::CountMinSketch; + +#[derive(Parser)] +#[command(about = "Generate deterministic Rust serialization snapshots")] +struct Arguments { + /// Directory in which snapshot files are written. + #[arg(long, default_value = "serialization/rust/snapshots")] + output: PathBuf, +} + +fn main() -> io::Result<()> { + let arguments = Arguments::parse(); + fs::create_dir_all(&arguments.output)?; + + write_snapshot( + &arguments.output, + "bloom_empty_rust.sk", + &BloomFilterBuilder::with_accuracy(128, 0.01) + .build() + .serialize(), + )?; + + let mut bloom = BloomFilterBuilder::with_accuracy(128, 0.01).build(); + for value in ["alpha", "beta", "gamma"] { + bloom.insert(value); + } + write_snapshot( + &arguments.output, + "bloom_non_empty_rust.sk", + &bloom.serialize(), + )?; + + let empty_countmin = CountMinSketch::::with_seed(4, 32, 9001); + write_snapshot( + &arguments.output, + "count_min_empty_rust.sk", + &empty_countmin.serialize(), + )?; + + let mut countmin = CountMinSketch::::with_seed(4, 32, 9001); + for (value, weight) in [("alpha", 3), ("beta", 2), ("gamma", 5)] { + for _ in 0..weight { + countmin.update(value); + } + } + write_snapshot( + &arguments.output, + "count_min_non_empty_rust.sk", + &countmin.serialize(), + )?; + + Ok(()) +} + +fn write_snapshot(output: &PathBuf, name: &str, bytes: &[u8]) -> io::Result<()> { + let path = output.join(name); + fs::write(&path, bytes)?; + println!("wrote {} ({} bytes)", path.display(), bytes.len()); + Ok(()) +} diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 50768fa..157dd8c 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -48,6 +48,7 @@ impl Command { SubCommand::Lint(cmd) => cmd.run(), SubCommand::Test(cmd) => cmd.run(), SubCommand::PrepareTestData(cmd) => cmd.run(), + SubCommand::GenerateSnapshots(cmd) => cmd.run(), } } } @@ -67,6 +68,11 @@ enum SubCommand { about = "Prepare serialization compatibility test data." )] PrepareTestData(CommandPrepareTestData), + #[clap( + name = "generate-snapshots", + about = "Generate deterministic Rust serialization snapshots." + )] + GenerateSnapshots(CommandGenerateSnapshots), } #[derive(Parser)] @@ -390,3 +396,15 @@ impl CommandPrepareTestData { languages } } + +#[derive(Parser)] +#[clap(name = "generate-snapshots")] +struct CommandGenerateSnapshots {} + +impl CommandGenerateSnapshots { + fn run(self) { + let mut command = find_command("cargo"); + command.args(["run", "--package", "snapshot_generator", "--"]); + run_command(command); + } +}