[rust] Check in generated protobuf code and drop protoc requirement

.github/workflows/rust-build-and-test.yml

@@ -62,11 +62,6 @@ jobs:
     steps:
       - uses: actions/checkout@v6
 
-      - name: Install protoc
-        uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0
-        with:
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-
       - name: Rust Cache
         uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
         with:
@@ -80,3 +75,25 @@ jobs:
         env:
           RUST_LOG: DEBUG
           RUST_BACKTRACE: full
+
+  proto-up-to-date:
+    timeout-minutes: 15
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Install protoc
+        uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Rust Cache
+        uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
+        with:
+          workspaces: fluss-rust
+
+      - name: Regenerate protobuf code
+        run: ./crates/fluss/regen.sh
+
+      - name: Verify the checked-in code is up to date
+        run: git diff --exit-code

.github/workflows/rust-license-and-format.yml

@@ -75,11 +75,6 @@ jobs:
       - name: Check dependency licenses (Apache-compatible)
         run: cargo deny check licenses
 
-      - name: Install protoc
-        uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0
-        with:
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-
       - name: Rust Cache
         uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
         with:

fluss-rust/Cargo.toml

@@ -28,7 +28,7 @@ keywords = ["fluss", "streaming-storage", "datalake"]
 
 [workspace]
 resolver = "2"
-members = ["crates/fluss", "crates/fluss-test-cluster", "crates/examples", "bindings/python", "bindings/cpp", "bindings/elixir/native/fluss_nif"]
+members = ["crates/fluss", "crates/fluss/gen", "crates/fluss-test-cluster", "crates/examples", "bindings/python", "bindings/cpp", "bindings/elixir/native/fluss_nif"]
 
 [workspace.dependencies]
 fluss = { package = "fluss-rs", version = "1.0.0", path = "crates/fluss", features = ["storage-all"] }

fluss-rust/DEVELOPMENT.md

@@ -4,16 +4,16 @@ Welcome to the development guide of `fluss-rust`! This project builds `fluss-rus
 
 ## Pre-requisites
 
-- protobuf
 - rust
 
 You can install these using your favourite package / version manager. Example installation using mise:
 
 ```bash
-mise install protobuf
 mise install rust
 ```
 
+`protobuf` (the `protoc` compiler) is only needed when regenerating the protobuf code after the canonical `FlussApi.proto` changes (see `crates/fluss/regen.sh`). Regular builds use the checked-in `crates/fluss/src/proto/fluss.rs`.
+
 ## IDE Setup
 
 We recommend [RustRover](https://www.jetbrains.com/rust/) IDE to work with fluss-rust code base.

fluss-rust/crates/fluss/Cargo.toml

@@ -86,6 +86,3 @@ fluss-test-cluster = { path = "../fluss-test-cluster" }
 # workspace `full` feature set; enable it for tests so the
 # `last_poll_seconds_ago` ticker loop can be driven deterministically.
 tokio = { workspace = true, features = ["test-util"] }
-
-[build-dependencies]
-prost-build = "0.14"

fluss-rust/crates/fluss/gen/Cargo.toml

@@ -0,0 +1,33 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[package]
+name = "gen"
+description = "Code generation for fluss-rs"
+version = "0.1.0"
+edition = { workspace = true }
+rust-version = { workspace = true }
+authors = { workspace = true }
+homepage = { workspace = true }
+repository = { workspace = true }
+license = { workspace = true }
+publish = false
+
+[dependencies]
+# Pin the prost-build version so regenerating does not churn the checked-in
+# src/proto/fluss.rs. Bump deliberately, then rerun ../regen.sh.
+prost-build = "0.14.3"

fluss-rust/crates/fluss/gen/src/main.rs

@@ -0,0 +1,77 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::error::Error;
+use std::fs;
+use std::path::Path;
+
+const HEADER: &str = "\
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// \"License\"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// \"AS IS\" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// This file was generated from the canonical FlussApi.proto by
+// fluss-rust/crates/fluss/regen.sh, and should not be edited by hand.
+
+";
+
+fn main() -> Result<(), Box<dyn Error>> {
+    let fluss_dir = Path::new(env!("CARGO_MANIFEST_DIR"))
+        .parent()
+        .ok_or("the gen crate must live inside crates/fluss")?;
+
+    // Published crates vendor the proto under proto/ (scripts/vendor-proto.sh);
+    // monorepo regeneration reads the canonical proto directly from fluss-rpc.
+    let vendored_dir = fluss_dir.join("proto");
+    let include_dir = if vendored_dir.join("FlussApi.proto").exists() {
+        vendored_dir
+    } else {
+        fluss_dir.join("../../../fluss-rpc/src/main/proto")
+    };
+    let proto = include_dir.join("FlussApi.proto");
+
+    let mut config = prost_build::Config::new();
+    config.bytes([
+        ".fluss.PbProduceLogReqForBucket.records",
+        ".fluss.PbPutKvReqForBucket.records",
+        ".fluss.PbLookupReqForBucket.keys",
+        ".fluss.PbPrefixLookupReqForBucket.keys",
+        ".fluss.ScanKvResponse.records",
+    ]);
+    let out_dir = fluss_dir.join("src/proto");
+    fs::create_dir_all(&out_dir)?;
+    config.out_dir(&out_dir);
+    config.compile_protos(&[&proto], &[&include_dir])?;
+
+    let generated_path = out_dir.join("fluss.rs");
+    let generated = fs::read_to_string(&generated_path)?;
+    fs::write(&generated_path, format!("{HEADER}{generated}"))?;
+    Ok(())
+}

fluss-rust/crates/fluss/regen.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Regenerates src/proto/fluss.rs from the canonical FlussApi.proto
+# (fluss-rpc/src/main/proto, or the vendored proto/ copy when present).
+# Requires protoc on PATH (or the PROTOC env var pointing at a protoc binary).
+
+set -euo pipefail
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+cd "$SCRIPT_DIR" && cargo run --manifest-path gen/Cargo.toml

fluss-rust/crates/fluss/src/lib.rs

@@ -152,5 +152,5 @@ pub(crate) mod proto {
     // caller yet, and the generated doc comments aren't clippy-clean.
     #![allow(dead_code)]
     #![allow(clippy::doc_lazy_continuation)]
-    include!(concat!(env!("OUT_DIR"), "/fluss.rs"));
+    include!("proto/fluss.rs");
 }