From 765a8a75b2a62366a3b77cbf72bbfb7aeb530e48 Mon Sep 17 00:00:00 2001 From: Wagner Bruna Date: Wed, 29 Jul 2026 22:22:26 -0300 Subject: [PATCH 1/6] sd: sync with master-801-9cfe2af --- otherarch/sdcpp/examples/common/common.cpp | 6 ++ .../sdcpp/src/conditioning/conditioner.hpp | 13 +++ otherarch/sdcpp/src/core/ggml_extend.hpp | 58 +++++++++--- otherarch/sdcpp/src/model/adapter/lora.hpp | 90 +++++++++++++++++-- otherarch/sdcpp/src/model/te/llm.hpp | 4 + otherarch/sdcpp/src/model_loader.cpp | 15 +--- otherarch/sdcpp/src/model_manager.cpp | 67 +++++++++++++- otherarch/sdcpp/src/model_manager.h | 4 +- otherarch/sdcpp/src/stable-diffusion.cpp | 33 +++++-- .../sdcpp/src/tokenizers/bpe_tokenizer.cpp | 2 +- 10 files changed, 253 insertions(+), 39 deletions(-) diff --git a/otherarch/sdcpp/examples/common/common.cpp b/otherarch/sdcpp/examples/common/common.cpp index 82558e7c24a1..d2ddca9f66c2 100644 --- a/otherarch/sdcpp/examples/common/common.cpp +++ b/otherarch/sdcpp/examples/common/common.cpp @@ -1901,6 +1901,7 @@ bool SDGenerationParams::from_json_str( load_if_exists("strength", strength); load_if_exists("control_strength", control_strength); + load_if_exists("ip_adapter_strength", ip_adapter_strength); load_if_exists("moe_boundary", moe_boundary); load_if_exists("vace_strength", vace_strength); @@ -2072,6 +2073,10 @@ bool SDGenerationParams::from_json_str( LOG_ERROR("invalid control_image"); return false; } + if (!parse_image_json_field(j, "ip_adapter_image", 3, width, height, ip_adapter_image)) { + LOG_ERROR("invalid ip_adapter_image"); + return false; + } return true; } @@ -2807,6 +2812,7 @@ std::string build_sdcpp_image_metadata_json(const SDContextParams& ctx_params, root["clip_skip"] = gen_params.clip_skip; root["strength"] = gen_params.strength; root["control_strength"] = gen_params.control_strength; + root["ip_adapter_strength"] = gen_params.ip_adapter_strength; root["auto_resize_ref_image"] = gen_params.auto_resize_ref_image; root["increase_ref_index"] = gen_params.increase_ref_index; if (mode == VID_GEN) { diff --git a/otherarch/sdcpp/src/conditioning/conditioner.hpp b/otherarch/sdcpp/src/conditioning/conditioner.hpp index e84e86297605..0b1f0d351a02 100644 --- a/otherarch/sdcpp/src/conditioning/conditioner.hpp +++ b/otherarch/sdcpp/src/conditioning/conditioner.hpp @@ -117,6 +117,7 @@ struct Conditioner { virtual SDCondition get_learned_condition(int n_threads, const ConditionerParams& conditioner_params) = 0; virtual void get_param_tensors(std::map& tensors) = 0; + virtual void get_param_tensor_ops(std::map& tensor_ops) {} virtual void set_max_graph_vram_bytes(size_t max_vram_bytes) {} virtual void set_stream_layers_enabled(bool enabled) {} virtual void set_runtime_backends(const std::vector& backends) {} @@ -1664,6 +1665,10 @@ struct AnimaConditioner : public Conditioner { llm->get_param_tensors(tensors, "text_encoders.llm"); } + void get_param_tensor_ops(std::map& tensor_ops) override { + llm->get_param_tensor_ops(tensor_ops); + } + void set_max_graph_vram_bytes(size_t max_vram_bytes) override { llm->set_max_graph_vram_bytes(max_vram_bytes); } @@ -1847,6 +1852,10 @@ struct LLMEmbedder : public Conditioner { } } + void get_param_tensor_ops(std::map& tensor_ops) override { + llm->get_param_tensor_ops(tensor_ops); + } + void set_max_graph_vram_bytes(size_t max_vram_bytes) override { llm->set_max_graph_vram_bytes(max_vram_bytes); if (byt5) { @@ -2828,6 +2837,10 @@ struct LTXAVEmbedder : public Conditioner { projector->get_param_tensors(tensors, "text_embedding_projection"); } + void get_param_tensor_ops(std::map& tensor_ops) override { + llm->get_param_tensor_ops(tensor_ops); + } + void set_flash_attention_enabled(bool enabled) override { llm->set_flash_attention_enabled(enabled); projector->set_flash_attention_enabled(enabled); diff --git a/otherarch/sdcpp/src/core/ggml_extend.hpp b/otherarch/sdcpp/src/core/ggml_extend.hpp index cc189f5b0cd9..53de509ad7e8 100644 --- a/otherarch/sdcpp/src/core/ggml_extend.hpp +++ b/otherarch/sdcpp/src/core/ggml_extend.hpp @@ -1753,7 +1753,7 @@ struct GGMLRunner { std::vector graph_cut_layer_split_backend_vram_limits_; std::vector extra_runtime_backends; // borrowed (SDBackendManager-owned) - ggml_backend_sched_t sched = nullptr; // owned, multi-device only + ggml_backend_sched_t sched = nullptr; // owned ggml_backend_t cpu_fallback_backend = nullptr; // owned, sched requires a trailing CPU backend bool multi_device_eval_callback_warned = false; @@ -2147,8 +2147,22 @@ struct GGMLRunner { return !extra_runtime_backends.empty(); } + bool graph_requires_backend_fallback(ggml_cgraph* gf) const { + if (gf == nullptr || sd_backend_is_cpu(runtime_backend)) { + return false; + } + const int n_nodes = ggml_graph_n_nodes(gf); + for (int i = 0; i < n_nodes; ++i) { + ggml_tensor* node = ggml_graph_node(gf, i); + if (node != nullptr && !ggml_backend_supports_op(runtime_backend, node)) { + return true; + } + } + return false; + } + bool alloc_compute_buffer(ggml_cgraph* gf) { - if (is_multi_device()) { + if (sched != nullptr || is_multi_device() || graph_requires_backend_fallback(gf)) { // The sched replaces the gallocr. Do NOT ggml_backend_sched_reserve // the graph here: reserve runs split_graph, which rewires the // graph's src pointers to sched-internal copy tensors, and the @@ -2156,6 +2170,10 @@ struct GGMLRunner { // rewired graph, silently corrupting every cross-backend input. A // graph must be split at most once; the alloc in execute_graph // performs the real allocation. + if (compute_allocr != nullptr) { + ggml_gallocr_free(compute_allocr); + compute_allocr = nullptr; + } return ensure_sched(gf); } if (compute_allocr != nullptr) { @@ -2753,7 +2771,7 @@ struct GGMLRunner { }; ComputeBufferGuard compute_buffer_guard(this, free_compute_buffer); - if (is_multi_device()) { + if (sched != nullptr) { ggml_backend_sched_reset(sched); pin_multi_device_nodes(gf); // reset clears the pins; re-apply before alloc if (!ggml_backend_sched_alloc_graph(sched, gf)) { @@ -2774,9 +2792,9 @@ struct GGMLRunner { } ggml_status status; - if (is_multi_device()) { + if (sched != nullptr) { if (sd_get_backend_eval_callback() != nullptr && !multi_device_eval_callback_warned) { - LOG_WARN("%s: eval callback is not supported with multiple runtime backends; ignoring", + LOG_WARN("%s: eval callback is not supported with the backend scheduler; ignoring", get_desc().c_str()); multi_device_eval_callback_warned = true; } @@ -3018,12 +3036,9 @@ struct GGMLRunner { // do copy after alloc graph void set_backend_tensor_data(ggml_tensor* tensor, const void* data) { - if (is_multi_device()) { - // The sched only assigns a backend (and thus a buffer) to tensors - // that participate in the graph; flag standalone data tensors as - // inputs so they get one. - ggml_set_input(tensor); - } + // The scheduler only allocates standalone data tensors when they are + // marked as graph inputs. The flag is harmless for single-backend graphs. + ggml_set_input(tensor); backend_tensor_data_map[tensor] = data; } @@ -3240,6 +3255,11 @@ class GGMLBlock { virtual void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, const std::string prefix = "") {} + virtual enum ggml_op param_usage_op(const std::string& name) const { + (void)name; + return GGML_OP_NONE; + } + public: void init(ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, std::string prefix = "") { if (prefix.size() > 0) { @@ -3290,6 +3310,18 @@ class GGMLBlock { } } + void get_param_tensor_ops(std::map& tensor_ops) { + for (auto& pair : blocks) { + pair.second->get_param_tensor_ops(tensor_ops); + } + for (auto& pair : params) { + enum ggml_op op = param_usage_op(pair.first); + if (op != GGML_OP_NONE) { + tensor_ops[pair.second] = op; + } + } + } + virtual std::string get_desc() { return "GGMLBlock"; } @@ -3417,6 +3449,10 @@ class Embedding : public UnaryBlock { params["weight"] = ggml_new_tensor_2d(ctx, wtype, embedding_dim, num_embeddings); } + enum ggml_op param_usage_op(const std::string& name) const override { + return name == "weight" ? GGML_OP_GET_ROWS : GGML_OP_NONE; + } + public: Embedding(int64_t num_embeddings, int64_t embedding_dim) : embedding_dim(embedding_dim), diff --git a/otherarch/sdcpp/src/model/adapter/lora.hpp b/otherarch/sdcpp/src/model/adapter/lora.hpp index 77b26ef994d4..e91ca0bd1cd2 100644 --- a/otherarch/sdcpp/src/model/adapter/lora.hpp +++ b/otherarch/sdcpp/src/model/adapter/lora.hpp @@ -14,6 +14,8 @@ struct LoraModel : public GGMLRunner { std::unordered_map lora_tensors; std::map original_tensor_to_final_tensor; std::set applied_lora_tensors; + std::set skipped_incompatible_lora_tensors; + std::set warned_incompatible_model_tensors; std::string file_path; std::shared_ptr model_manager; ggml_backend_t params_backend = nullptr; @@ -133,6 +135,8 @@ struct LoraModel : public GGMLRunner { lora_tensors.clear(); original_tensor_to_final_tensor.clear(); applied_lora_tensors.clear(); + skipped_incompatible_lora_tensors.clear(); + warned_incompatible_model_tensors.clear(); applied = false; tensor_preprocessed = false; } @@ -338,7 +342,9 @@ struct LoraModel : public GGMLRunner { iter = lora_tensors.find(hada_1_mid_name); if (iter != lora_tensors.end()) { hada_1_mid = ggml_ext_cast_f32(ctx, backend, iter->second); - hada_1_up = ggml_cont(ctx, ggml_transpose(ctx, hada_1_up)); + if (hada_1_up != nullptr) { + hada_1_up = ggml_cont(ctx, ggml_transpose(ctx, hada_1_up)); + } } iter = lora_tensors.find(hada_2_down_name); @@ -354,7 +360,9 @@ struct LoraModel : public GGMLRunner { iter = lora_tensors.find(hada_2_mid_name); if (iter != lora_tensors.end()) { hada_2_mid = ggml_ext_cast_f32(ctx, backend, iter->second); - hada_2_up = ggml_cont(ctx, ggml_transpose(ctx, hada_2_up)); + if (hada_2_up != nullptr) { + hada_2_up = ggml_cont(ctx, ggml_transpose(ctx, hada_2_up)); + } } if (hada_1_up == nullptr || hada_1_down == nullptr || hada_2_up == nullptr || hada_2_down == nullptr) { @@ -546,7 +554,27 @@ struct LoraModel : public GGMLRunner { } } - GGML_ASSERT(ggml_nelements(diff) == ggml_nelements(model_tensor)); + if (ggml_nelements(diff) != ggml_nelements(model_tensor)) { + const std::string lora_tensor_prefix = "lora." + model_tensor_name + "."; + for (const auto& tensor_name : applied_lora_tensors) { + if (starts_with(tensor_name, lora_tensor_prefix)) { + skipped_incompatible_lora_tensors.insert(tensor_name); + } + } + if (warned_incompatible_model_tensors.insert(model_tensor_name).second) { + LOG_WARN("skip incompatible LoRA tensor |%s|: model shape = [%lld, %lld, %lld, %lld], LoRA shape = [%lld, %lld, %lld, %lld]", + model_tensor_name.c_str(), + static_cast(model_tensor->ne[0]), + static_cast(model_tensor->ne[1]), + static_cast(model_tensor->ne[2]), + static_cast(model_tensor->ne[3]), + static_cast(diff->ne[0]), + static_cast(diff->ne[1]), + static_cast(diff->ne[2]), + static_cast(diff->ne[3])); + } + return nullptr; + } diff = ggml_reshape(ctx, diff, model_tensor); } return diff; @@ -555,6 +583,7 @@ struct LoraModel : public GGMLRunner { ggml_tensor* get_out_diff(ggml_context* ctx, ggml_backend_t backend, ggml_tensor* x, + ggml_tensor* model_weight, WeightAdapter::ForwardParams forward_params, const std::string& model_tensor_name) { ggml_tensor* out_diff = nullptr; @@ -707,6 +736,43 @@ struct LoraModel : public GGMLRunner { break; } + if (!is_conv2d) { + const int64_t down_in = lora_down->ne[0]; + const int64_t down_out = lora_down->ne[1]; + const int64_t up_in = lora_up->ne[0]; + const int64_t up_out = lora_up->ne[1]; + + bool compatible = down_in == model_weight->ne[0] && + up_out == model_weight->ne[1]; + if (lora_mid != nullptr) { + compatible = compatible && + lora_mid->ne[0] == down_out && + up_in == lora_mid->ne[1]; + } else { + compatible = compatible && up_in == down_out; + } + + if (!compatible) { + skipped_incompatible_lora_tensors.insert(lora_down_name); + skipped_incompatible_lora_tensors.insert(lora_up_name); + skipped_incompatible_lora_tensors.insert(lora_mid_name); + skipped_incompatible_lora_tensors.insert(scale_name); + skipped_incompatible_lora_tensors.insert(alpha_name); + if (warned_incompatible_model_tensors.insert(model_tensor_name).second) { + LOG_WARN("skip incompatible LoRA tensor |%s|: model shape = [%lld, %lld], down shape = [%lld, %lld], up shape = [%lld, %lld]", + model_tensor_name.c_str(), + static_cast(model_weight->ne[0]), + static_cast(model_weight->ne[1]), + static_cast(down_in), + static_cast(down_out), + static_cast(up_in), + static_cast(up_out)); + } + index++; + continue; + } + } + applied_lora_tensors.insert(lora_up_name); applied_lora_tensors.insert(lora_down_name); @@ -869,10 +935,13 @@ struct LoraModel : public GGMLRunner { void stat(bool at_runntime = false) { size_t total_lora_tensors_count = 0; size_t applied_lora_tensors_count = 0; + size_t skipped_lora_tensors_count = 0; for (auto& kv : lora_tensors) { total_lora_tensors_count++; - if (applied_lora_tensors.find(kv.first) == applied_lora_tensors.end()) { + if (skipped_incompatible_lora_tensors.find(kv.first) != skipped_incompatible_lora_tensors.end()) { + skipped_lora_tensors_count++; + } else if (applied_lora_tensors.find(kv.first) == applied_lora_tensors.end()) { if (!at_runntime) { LOG_WARN("unused lora tensor |%s|", kv.first.c_str()); //print_ggml_tensor(kv.second, true); @@ -884,12 +953,17 @@ struct LoraModel : public GGMLRunner { /* Don't worry if this message shows up twice in the logs per LoRA, * this function is called once to calculate the required buffer size * and then again to actually generate a graph to be used */ - if (!at_runntime && applied_lora_tensors_count != total_lora_tensors_count) { + size_t compatible_lora_tensors_count = total_lora_tensors_count - skipped_lora_tensors_count; + if (!at_runntime && applied_lora_tensors_count != compatible_lora_tensors_count) { LOG_WARN("Only (%lu / %lu) LoRA tensors have been applied, lora_file_path = %s", - applied_lora_tensors_count, total_lora_tensors_count, file_path.c_str()); + applied_lora_tensors_count, compatible_lora_tensors_count, file_path.c_str()); } else { LOG_INFO("(%lu / %lu) LoRA tensors have been applied, lora_file_path = %s", - applied_lora_tensors_count, total_lora_tensors_count, file_path.c_str()); + applied_lora_tensors_count, compatible_lora_tensors_count, file_path.c_str()); + } + if (skipped_lora_tensors_count > 0) { + LOG_WARN("(%lu / %lu) incompatible LoRA tensors have been skipped, lora_file_path = %s", + skipped_lora_tensors_count, total_lora_tensors_count, file_path.c_str()); } } }; @@ -953,7 +1027,7 @@ struct MultiLoraAdapter : public WeightAdapter { forward_params.conv2d.scale); } for (auto& lora_model : lora_models) { - ggml_tensor* out_diff = lora_model->get_out_diff(ctx, backend, x, forward_params, prefix + "weight"); + ggml_tensor* out_diff = lora_model->get_out_diff(ctx, backend, x, w, forward_params, prefix + "weight"); if (out_diff == nullptr) { continue; } diff --git a/otherarch/sdcpp/src/model/te/llm.hpp b/otherarch/sdcpp/src/model/te/llm.hpp index 60e2d9c15c72..c0651a4a80b1 100644 --- a/otherarch/sdcpp/src/model/te/llm.hpp +++ b/otherarch/sdcpp/src/model/te/llm.hpp @@ -1657,6 +1657,10 @@ namespace LLM { model.get_param_tensors(tensors, prefix); } + void get_param_tensor_ops(std::map& tensor_ops) { + model.get_param_tensor_ops(tensor_ops); + } + ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* input_ids, ggml_tensor* input_pos, diff --git a/otherarch/sdcpp/src/model_loader.cpp b/otherarch/sdcpp/src/model_loader.cpp index b76dd8e49162..ae8c03871895 100644 --- a/otherarch/sdcpp/src/model_loader.cpp +++ b/otherarch/sdcpp/src/model_loader.cpp @@ -1083,7 +1083,7 @@ bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb, std::atomic tensor_idx(0); std::atomic failed(false); std::vector workers; - std::mutex rpc_backend_mutex; + std::mutex backend_tensor_set_mutex; for (int i = 0; i < n_threads; ++i) { workers.emplace_back([&, file_path, is_zip]() { @@ -1271,17 +1271,8 @@ bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb, if (dst_tensor->buffer != nullptr && !ggml_backend_buffer_is_host(dst_tensor->buffer)) { t0 = ggml_time_ms(); - // RPC backends require serialized access to prevent concurrency issues - const char* buffer_type_name = ggml_backend_buft_name(ggml_backend_buffer_get_type(dst_tensor->buffer)); - bool is_rpc_buffer = buffer_type_name != nullptr && - std::string(buffer_type_name).find("RPC") != std::string::npos; - - if (is_rpc_buffer) { - std::lock_guard lock(rpc_backend_mutex); - ggml_backend_tensor_set(dst_tensor, convert_buf, 0, ggml_nbytes(dst_tensor)); - } else { - ggml_backend_tensor_set(dst_tensor, convert_buf, 0, ggml_nbytes(dst_tensor)); - } + std::lock_guard lock(backend_tensor_set_mutex); + ggml_backend_tensor_set(dst_tensor, convert_buf, 0, ggml_nbytes(dst_tensor)); t1 = ggml_time_ms(); copy_to_backend_time_ms.fetch_add(t1 - t0); diff --git a/otherarch/sdcpp/src/model_manager.cpp b/otherarch/sdcpp/src/model_manager.cpp index 33bc1837310b..2f44b0003b31 100644 --- a/otherarch/sdcpp/src/model_manager.cpp +++ b/otherarch/sdcpp/src/model_manager.cpp @@ -53,6 +53,48 @@ static bool backend_supports_host_buffer(ggml_backend_t backend) { return props.caps.buffer_from_host_ptr; } +static bool device_supports_param_op(ggml_backend_dev_t device, + ggml_tensor* weight, + enum ggml_op op, + ggml_backend_buffer_type_t buft) { + if (op == GGML_OP_NONE) { + return true; + } + if (device == nullptr || weight == nullptr || buft == nullptr || weight->buffer != nullptr) { + return false; + } + + ggml_init_params params; + params.mem_size = ggml_tensor_overhead() * 2; + params.mem_buffer = nullptr; + params.no_alloc = true; + ggml_context* ctx = ggml_init(params); + if (ctx == nullptr) { + return false; + } + + ggml_tensor* op_tensor = nullptr; + if (op == GGML_OP_GET_ROWS) { + ggml_tensor* indices = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, 1); + op_tensor = ggml_get_rows(ctx, weight, indices); + } + if (op_tensor == nullptr) { + ggml_free(ctx); + return false; + } + + weight->buffer = ggml_backend_buft_alloc_buffer(buft, 0); + if (weight->buffer == nullptr) { + ggml_free(ctx); + return false; + } + bool supported = ggml_backend_dev_supports_op(device, op_tensor); + ggml_backend_buffer_free(weight->buffer); + weight->buffer = nullptr; + ggml_free(ctx); + return supported; +} + ModelManager::~ModelManager() { release_all(); } @@ -135,7 +177,8 @@ bool ModelManager::register_param_tensors(const std::string& desc, ggml_backend_t params_backend, size_t* registered_tensor_size, bool allow_split_buffer, - bool params_follow_compute_backend) { + bool params_follow_compute_backend, + const std::map* tensor_ops) { if (desc.empty()) { LOG_ERROR("model manager tensor desc is empty"); return false; @@ -168,6 +211,12 @@ bool ModelManager::register_param_tensors(const std::string& desc, state->params_backend = params_backend; state->allow_split_buffer = allow_split_buffer; state->params_follow_compute_backend = params_follow_compute_backend; + if (tensor_ops != nullptr) { + auto op_it = tensor_ops->find(tensor); + if (op_it != tensor_ops->end()) { + state->usage_op = op_it->second; + } + } new_states.push_back(std::move(state)); } @@ -844,6 +893,22 @@ ggml_backend_buffer_type_t ModelManager::params_buffer_type_for(const TensorStat if (params_buft == nullptr) { params_buft = ggml_backend_get_default_buffer_type(state.params_backend); } + if (state.usage_op != GGML_OP_NONE && + state.compute_backend != nullptr) { + ggml_backend_dev_t compute_dev = ggml_backend_get_device(state.compute_backend); + if (device_supports_param_op(compute_dev, state.tensor, state.usage_op, params_buft)) { + return params_buft; + } + + ggml_backend_dev_t cpu_dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU); + params_buft = cpu_dev != nullptr ? ggml_backend_dev_buffer_type(cpu_dev) : nullptr; + if (!device_supports_param_op(cpu_dev, state.tensor, state.usage_op, params_buft)) { + LOG_ERROR("model manager has no compatible buffer for tensor '%s' used by %s", + state.name.c_str(), + ggml_op_name(state.usage_op)); + return nullptr; + } + } return params_buft; } diff --git a/otherarch/sdcpp/src/model_manager.h b/otherarch/sdcpp/src/model_manager.h index 85e982de2b93..881a17714378 100644 --- a/otherarch/sdcpp/src/model_manager.h +++ b/otherarch/sdcpp/src/model_manager.h @@ -39,6 +39,7 @@ class ModelManager : public RunnerWeightManager { bool allow_split_buffer = false; bool params_follow_compute_backend = false; bool metadata_validated = false; + enum ggml_op usage_op = GGML_OP_NONE; int active_prepare_count = 0; @@ -132,7 +133,8 @@ class ModelManager : public RunnerWeightManager { ggml_backend_t params_backend, size_t* registered_tensor_size = nullptr, bool allow_split_buffer = false, - bool params_follow_compute_backend = false); + bool params_follow_compute_backend = false, + const std::map* tensor_ops = nullptr); bool unregister_param_tensors(const std::string& desc, size_t* registered_tensor_size = nullptr); diff --git a/otherarch/sdcpp/src/stable-diffusion.cpp b/otherarch/sdcpp/src/stable-diffusion.cpp index 1f2b8b4d231d..ef8b17b38066 100644 --- a/otherarch/sdcpp/src/stable-diffusion.cpp +++ b/otherarch/sdcpp/src/stable-diffusion.cpp @@ -143,6 +143,8 @@ const char* sampling_methods_str[] = { "Euler CFG++", "Euler A CFG++", "Euler GE", + "DPM++ (2M) SDE", + "DPM++ (2M) SDE BT", }; /*================================================== Helper Functions ================================================*/ @@ -322,7 +324,11 @@ class StableDiffusionGGML { return true; } std::map group_tensors; + std::map tensor_ops; model->get_param_tensors(group_tensors); + if constexpr (std::is_base_of_v) { + model->get_param_tensor_ops(tensor_ops); + } if (model_manager == nullptr) { return true; } @@ -339,6 +345,7 @@ class StableDiffusionGGML { module, module_backends, std::move(group_tensors), + tensor_ops, residency_mode, params_mem_size); } @@ -347,6 +354,7 @@ class StableDiffusionGGML { module, module_backends, std::move(group_tensors), + tensor_ops, residency_mode, params_mem_size); } @@ -360,7 +368,10 @@ class StableDiffusionGGML { residency_mode, backend_for(module), params_backend_for(module), - params_mem_size); + params_mem_size, + false, + false, + &tensor_ops); } template @@ -369,6 +380,7 @@ class StableDiffusionGGML { SDBackendModule module, const std::vector& module_backends, std::map group_tensors, + const std::map& tensor_ops, ModelManager::ResidencyMode residency_mode, size_t* params_mem_size) { ggml_backend_t main_backend = module_backends[0]; @@ -380,6 +392,7 @@ class StableDiffusionGGML { module, module_backends, std::move(group_tensors), + tensor_ops, residency_mode, params_mem_size); }; @@ -454,7 +467,9 @@ class StableDiffusionGGML { main_backend, params_backend_for(module), params_mem_size, - /*allow_split_buffer=*/true)) { + /*allow_split_buffer=*/true, + false, + &tensor_ops)) { return false; } return model_manager->register_param_tensors(desc, @@ -462,7 +477,10 @@ class StableDiffusionGGML { residency_mode, main_backend, params_backend_for(module), - params_mem_size); + params_mem_size, + false, + false, + &tensor_ops); } // Register graph-cut layer-split tensors on the primary backend first. @@ -474,6 +492,7 @@ class StableDiffusionGGML { SDBackendModule module, const std::vector& module_backends, std::map group_tensors, + const std::map& tensor_ops, ModelManager::ResidencyMode residency_mode, size_t* params_mem_size) { bool has_cpu_device = false; @@ -495,7 +514,10 @@ class StableDiffusionGGML { residency_mode, module_backends[0], params_backend_for(module), - params_mem_size); + params_mem_size, + false, + false, + &tensor_ops); } model->set_runtime_backends(module_backends); @@ -520,7 +542,8 @@ class StableDiffusionGGML { initial_params_backend, params_mem_size, false, - params_follow_runtime); + params_follow_runtime, + &tensor_ops); } bool unload_control_net() { diff --git a/otherarch/sdcpp/src/tokenizers/bpe_tokenizer.cpp b/otherarch/sdcpp/src/tokenizers/bpe_tokenizer.cpp index 7733f00d36b7..2858e346f602 100644 --- a/otherarch/sdcpp/src/tokenizers/bpe_tokenizer.cpp +++ b/otherarch/sdcpp/src/tokenizers/bpe_tokenizer.cpp @@ -205,7 +205,7 @@ std::vector BPETokenizer::encode(const std::string& text, on_new_token_cb_t ss << "\"" << token << "\", "; } ss << "]"; - LOG_DEBUG("split prompt \"%s\" to tokens %s", text.c_str(), ss.str().c_str()); + LOG_DEBUG("split prompt \"%s\" to %zu tokens %s", text.c_str(), bpe_tokens.size(), ss.str().c_str()); return bpe_tokens; } From b53ced8d6a64bfd6e75ef5e523d9e86168c19bf2 Mon Sep 17 00:00:00 2001 From: Wagner Bruna Date: Thu, 30 Jul 2026 00:06:22 -0300 Subject: [PATCH 2/6] sd: sync with master-802-e92e86f --- otherarch/sdcpp/src/model_io/pickle_io.cpp | 65 +++++++++++++++++-- .../sdcpp/src/model_io/torch_legacy_io.cpp | 17 +++-- otherarch/sdcpp/src/model_io/torch_zip_io.cpp | 6 +- otherarch/sdcpp/src/model_loader.cpp | 52 +++++++++++---- otherarch/sdcpp/src/model_manager.cpp | 6 +- otherarch/sdcpp/src/model_manager.h | 6 +- 6 files changed, 123 insertions(+), 29 deletions(-) diff --git a/otherarch/sdcpp/src/model_io/pickle_io.cpp b/otherarch/sdcpp/src/model_io/pickle_io.cpp index 1ea7a902c93f..a597e66b3890 100644 --- a/otherarch/sdcpp/src/model_io/pickle_io.cpp +++ b/otherarch/sdcpp/src/model_io/pickle_io.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -512,8 +513,51 @@ static bool parse_storage_type(const std::string& global_name, PickleStorageInfo return false; } -static bool tensor_is_contiguous(const PickleTensorInfo& tensor) { - if (tensor.tensor_storage.nelements() == 0) { +static bool checked_pickle_byte_count(int64_t element_count, + uint64_t element_nbytes, + uint64_t* byte_count) { + if (element_count < 0 || element_nbytes == 0) { + return false; + } + + uint64_t count = static_cast(element_count); + if (count > std::numeric_limits::max() / element_nbytes) { + return false; + } + + *byte_count = count * element_nbytes; + return true; +} + +static bool tensor_layout_is_valid(const PickleTensorInfo& tensor, uint64_t raw_element_nbytes) { + if (raw_element_nbytes == 0) { + return false; + } + + bool has_zero_dimension = false; + uint64_t element_count = 1; + for (int i = 0; i < tensor.tensor_storage.n_dims; ++i) { + int64_t dimension = tensor.tensor_storage.ne[i]; + if (dimension < 0) { + return false; + } + if (dimension == 0) { + has_zero_dimension = true; + continue; + } + + uint64_t size = static_cast(dimension); + if (element_count > static_cast(std::numeric_limits::max()) / size) { + return false; + } + element_count *= size; + } + + if (!has_zero_dimension && + element_count > static_cast(std::numeric_limits::max()) / raw_element_nbytes) { + return false; + } + if (has_zero_dimension) { return true; } if (tensor.stride_n_dims != tensor.tensor_storage.n_dims) { @@ -932,7 +976,12 @@ bool parse_torch_state_dict_pickle(const uint8_t* buffer, if (storage.key.empty() || !parse_storage_type(pid.items[1].str_value, &storage)) { return false; } - storage.nbytes = (uint64_t)pid.items[4].int_value * storage.raw_element_nbytes; + if (!checked_pickle_byte_count(pid.items[4].int_value, + storage.raw_element_nbytes, + &storage.nbytes)) { + set_error(error, "invalid storage size in torch pickle"); + return false; + } storage_nbytes[storage.key] = storage.nbytes; stack.push_back(make_storage_value(storage)); } break; @@ -963,7 +1012,12 @@ bool parse_torch_state_dict_pickle(const uint8_t* buffer, tensor.tensor_storage.is_f64 = args.items[0].storage.is_f64; tensor.tensor_storage.is_i64 = args.items[0].storage.is_i64; tensor.tensor_storage.storage_key = args.items[0].storage.key; - tensor.tensor_storage.offset = (uint64_t)args.items[1].int_value * args.items[0].storage.raw_element_nbytes; + if (!checked_pickle_byte_count(args.items[1].int_value, + args.items[0].storage.raw_element_nbytes, + &tensor.tensor_storage.offset)) { + set_error(error, "invalid tensor storage offset in torch pickle"); + return false; + } for (const auto& item : args.items[2].items) { if (item.kind != PickleValue::INT || tensor.tensor_storage.n_dims >= SD_MAX_DIMS) { @@ -979,7 +1033,8 @@ bool parse_torch_state_dict_pickle(const uint8_t* buffer, tensor.stride[tensor.stride_n_dims++] = item.int_value; } - if (!tensor_is_contiguous(tensor)) { + if (!tensor_layout_is_valid(tensor, args.items[0].storage.raw_element_nbytes)) { + set_error(error, "invalid tensor shape or stride in torch pickle"); return false; } stack.push_back(make_tensor_value(tensor)); diff --git a/otherarch/sdcpp/src/model_io/torch_legacy_io.cpp b/otherarch/sdcpp/src/model_io/torch_legacy_io.cpp index 0b0c78f8fdc8..3170fe8d4c72 100644 --- a/otherarch/sdcpp/src/model_io/torch_legacy_io.cpp +++ b/otherarch/sdcpp/src/model_io/torch_legacy_io.cpp @@ -139,11 +139,16 @@ bool read_torch_legacy_file(const std::string& file_path, if (it == legacy_storage_map.end()) { return false; } - if (current_offset + LEGACY_STORAGE_HEADER_SIZE + it->second > file_size) { + if (current_offset > file_size || + LEGACY_STORAGE_HEADER_SIZE > file_size - current_offset) { return false; } - storage_offsets[storage_key] = current_offset + LEGACY_STORAGE_HEADER_SIZE; - current_offset += LEGACY_STORAGE_HEADER_SIZE + it->second; + uint64_t storage_offset = current_offset + LEGACY_STORAGE_HEADER_SIZE; + if (it->second > file_size - storage_offset) { + return false; + } + storage_offsets[storage_key] = storage_offset; + current_offset = storage_offset + it->second; } for (auto& tensor_storage : tensor_storages) { @@ -159,8 +164,10 @@ bool read_torch_legacy_file(const std::string& file_path, uint64_t base_offset = it_offset->second; uint64_t storage_nbytes = it_size->second; - uint64_t tensor_nbytes = tensor_storage.nbytes_to_read(); - if (tensor_storage.offset + tensor_nbytes > storage_nbytes) { + int64_t tensor_nbytes = tensor_storage.nbytes_to_read(); + if (tensor_nbytes < 0 || + tensor_storage.offset > storage_nbytes || + static_cast(tensor_nbytes) > storage_nbytes - tensor_storage.offset) { return false; } diff --git a/otherarch/sdcpp/src/model_io/torch_zip_io.cpp b/otherarch/sdcpp/src/model_io/torch_zip_io.cpp index 9eaf6c53a396..a5db29a11376 100644 --- a/otherarch/sdcpp/src/model_io/torch_zip_io.cpp +++ b/otherarch/sdcpp/src/model_io/torch_zip_io.cpp @@ -76,8 +76,10 @@ static bool parse_zip_data_pkl(const uint8_t* buffer, return false; } - uint64_t tensor_nbytes = tensor_storage.nbytes_to_read(); - if (tensor_storage.offset + tensor_nbytes > entry_size) { + int64_t tensor_nbytes = tensor_storage.nbytes_to_read(); + if (tensor_nbytes < 0 || + tensor_storage.offset > entry_size || + static_cast(tensor_nbytes) > entry_size - tensor_storage.offset) { set_error(error, "tensor '" + tensor_storage.name + "' exceeds storage entry '" + entry_name + "'"); return false; } diff --git a/otherarch/sdcpp/src/model_loader.cpp b/otherarch/sdcpp/src/model_loader.cpp index ae8c03871895..582cca036595 100644 --- a/otherarch/sdcpp/src/model_loader.cpp +++ b/otherarch/sdcpp/src/model_loader.cpp @@ -1107,6 +1107,7 @@ bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb, std::vector read_buffer; std::vector convert_buffer; + std::vector zip_entry_buffer; std::vector dequant_buffer; std::vector scale_buffer; @@ -1147,37 +1148,63 @@ bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb, size_t nbytes_to_read = tensor_storage.nbytes_to_read(); - auto read_data_at = [&](char* buf, size_t n, uint64_t offset) { + auto read_data_at = [&](char* buf, size_t n, uint64_t offset) -> bool { if (zip != nullptr) { - zip_entry_openbyindex(zip, tensor_storage.index_in_zip); + if (zip_entry_openbyindex(zip, tensor_storage.index_in_zip) != 0) { + LOG_ERROR("failed to open zip entry for tensor '%s'", tensor_storage.name.c_str()); + return false; + } size_t entry_size = zip_entry_size(zip); + if (offset > entry_size) { + LOG_ERROR("tensor '%s' exceeds its zip storage entry", tensor_storage.name.c_str()); + zip_entry_close(zip); + return false; + } + size_t tensor_offset = static_cast(offset); + if (n > entry_size - tensor_offset) { + LOG_ERROR("tensor '%s' exceeds its zip storage entry", tensor_storage.name.c_str()); + zip_entry_close(zip); + return false; + } + if (entry_size != n || offset != 0) { int64_t t_memcpy_start; - read_buffer.resize(entry_size); - zip_entry_noallocread(zip, (void*)read_buffer.data(), entry_size); + zip_entry_buffer.resize(entry_size); + auto bytes_read = zip_entry_noallocread(zip, (void*)zip_entry_buffer.data(), entry_size); + if (bytes_read < 0 || static_cast(bytes_read) != entry_size) { + LOG_ERROR("failed to read zip entry for tensor '%s'", tensor_storage.name.c_str()); + zip_entry_close(zip); + return false; + } t_memcpy_start = ggml_time_ms(); - memcpy((void*)buf, (void*)(read_buffer.data() + offset), n); + memcpy((void*)buf, (void*)(zip_entry_buffer.data() + tensor_offset), n); memcpy_time_ms.fetch_add(ggml_time_ms() - t_memcpy_start); } else { - zip_entry_noallocread(zip, (void*)buf, n); + auto bytes_read = zip_entry_noallocread(zip, (void*)buf, n); + if (bytes_read < 0 || static_cast(bytes_read) != n) { + LOG_ERROR("failed to read zip entry for tensor '%s'", tensor_storage.name.c_str()); + zip_entry_close(zip); + return false; + } } zip_entry_close(zip); } else if (mmapped) { if (!mmapped->copy_data(buf, n, offset)) { LOG_ERROR("read tensor data failed: '%s'", file_path.c_str()); - failed = true; + return false; } } else { file.seekg(offset); file.read(buf, n); if (!file) { LOG_ERROR("read tensor data failed: '%s'", file_path.c_str()); - failed = true; + return false; } } + return true; }; - auto read_data = [&](char* buf, size_t n) { - read_data_at(buf, n, tensor_storage.offset); + auto read_data = [&](char* buf, size_t n) -> bool { + return read_data_at(buf, n, tensor_storage.offset); }; char* read_buf = nullptr; @@ -1219,7 +1246,10 @@ bool ModelLoader::load_tensors(on_new_tensor_cb_t on_new_tensor_cb, } t0 = ggml_time_ms(); - read_data(read_buf, nbytes_to_read); + if (!read_data(read_buf, nbytes_to_read)) { + failed = true; + break; + } t1 = ggml_time_ms(); read_time_ms.fetch_add(t1 - t0); diff --git a/otherarch/sdcpp/src/model_manager.cpp b/otherarch/sdcpp/src/model_manager.cpp index 2f44b0003b31..8db303dda956 100644 --- a/otherarch/sdcpp/src/model_manager.cpp +++ b/otherarch/sdcpp/src/model_manager.cpp @@ -54,9 +54,9 @@ static bool backend_supports_host_buffer(ggml_backend_t backend) { } static bool device_supports_param_op(ggml_backend_dev_t device, - ggml_tensor* weight, - enum ggml_op op, - ggml_backend_buffer_type_t buft) { + ggml_tensor* weight, + enum ggml_op op, + ggml_backend_buffer_type_t buft) { if (op == GGML_OP_NONE) { return true; } diff --git a/otherarch/sdcpp/src/model_manager.h b/otherarch/sdcpp/src/model_manager.h index 881a17714378..3dc6338645a2 100644 --- a/otherarch/sdcpp/src/model_manager.h +++ b/otherarch/sdcpp/src/model_manager.h @@ -131,9 +131,9 @@ class ModelManager : public RunnerWeightManager { ResidencyMode residency_mode, ggml_backend_t compute_backend, ggml_backend_t params_backend, - size_t* registered_tensor_size = nullptr, - bool allow_split_buffer = false, - bool params_follow_compute_backend = false, + size_t* registered_tensor_size = nullptr, + bool allow_split_buffer = false, + bool params_follow_compute_backend = false, const std::map* tensor_ops = nullptr); bool unregister_param_tensors(const std::string& desc, From 6ada2b474168c854edd13e20e9a3524360af8597 Mon Sep 17 00:00:00 2001 From: Wagner Bruna Date: Thu, 30 Jul 2026 21:57:25 -0300 Subject: [PATCH 3/6] sd: sync with master-805-e31a86c --- otherarch/sdcpp/examples/common/common.cpp | 2 +- .../src/extensions/photomaker_extension.cpp | 2 +- otherarch/sdcpp/src/name_conversion.cpp | 9 ++++++ otherarch/sdcpp/src/runtime/denoiser.hpp | 31 +++++++++++++++++-- otherarch/sdcpp/src/stable-diffusion.cpp | 11 ++----- 5 files changed, 42 insertions(+), 13 deletions(-) diff --git a/otherarch/sdcpp/examples/common/common.cpp b/otherarch/sdcpp/examples/common/common.cpp index d2ddca9f66c2..ceb7814308e2 100644 --- a/otherarch/sdcpp/examples/common/common.cpp +++ b/otherarch/sdcpp/examples/common/common.cpp @@ -1008,7 +1008,7 @@ ArgOptions SDGenerationParams::get_options() { &hires_upscaler}, {"", "--extra-sample-args", - "extra sampler/scheduler/guidance args, key=value list. CFG supports guidance_schedule; APG supports apg_eta, apg_momentum, apg_norm_threshold, apg_norm_threshold_smoothing; SLG supports slg_uncond; lcm supports noise_clip_std, noise_scale_start, noise_scale_end; flux supports base_shift, max_shift; ltx2 supports max_shift, base_shift, stretch, terminal; euler_ge supports gamma;; logit_normal supports mu, std, logsnr_min, logsnr_max, resolution_aware", + "extra sampler/scheduler/guidance args, key=value list. CFG supports guidance_schedule; APG supports apg_eta, apg_momentum, apg_norm_threshold, apg_norm_threshold_smoothing; SLG supports slg_uncond; lcm supports noise_clip_std, noise_scale_start, noise_scale_end; flux supports base_shift, max_shift; ltx2 supports max_shift, base_shift, stretch, terminal; euler_ge supports gamma; beta scheduler supports alpha, beta; logit_normal supports mu, std, logsnr_min, logsnr_max, resolution_aware", (int)',', &extra_sample_args}, {"", diff --git a/otherarch/sdcpp/src/extensions/photomaker_extension.cpp b/otherarch/sdcpp/src/extensions/photomaker_extension.cpp index 78c5cdb9b749..48b023f99367 100644 --- a/otherarch/sdcpp/src/extensions/photomaker_extension.cpp +++ b/otherarch/sdcpp/src/extensions/photomaker_extension.cpp @@ -56,7 +56,7 @@ tokenize_photomaker_trigger(FrozenCLIPEmbedderWithCustomWords& clip_conditioner, true); std::vector class_token_mask; for (int i = 0; i < tokens.size(); i++) { - class_token_mask.push_back(class_idx + 1 <= i && i < class_idx + 1 + trigger_token_count); + class_token_mask.push_back(class_idx >= 0 && class_idx + 1 <= i && i < class_idx + 1 + trigger_token_count); } return std::make_tuple(tokens, weights, class_token_mask); diff --git a/otherarch/sdcpp/src/name_conversion.cpp b/otherarch/sdcpp/src/name_conversion.cpp index 63c916edd85a..af8a7f572d1f 100644 --- a/otherarch/sdcpp/src/name_conversion.cpp +++ b/otherarch/sdcpp/src/name_conversion.cpp @@ -1449,12 +1449,21 @@ std::string convert_tensor_name(std::string name, SDVersion version) { {"te2.", "cond_stage_model.1.transformer."}, {"te1.", "cond_stage_model.transformer."}, {"te3.", "text_encoders.t5xxl.transformer."}, + {"clip_vision.", "cond_stage_model.transformer."}, }; if (sd_version_is_flux(version)) { prefix_map["te1."] = "text_encoders.clip_l.transformer."; } + if (sd_version_is_unet(version)) { + prefix_map["clip_l."] = "cond_stage_model.transformer."; + prefix_map["clip_g."] = "cond_stage_model.1.transformer."; + } else { + prefix_map["clip_l."] = "text_encoders.clip_l.transformer."; + prefix_map["clip_g."] = "text_encoders.clip_g.transformer."; + } + replace_with_prefix_map(name, prefix_map); if (sd_version_is_boogu_image(version) || sd_version_is_krea2(version) || sd_version_is_mage_flow(version)) { diff --git a/otherarch/sdcpp/src/runtime/denoiser.hpp b/otherarch/sdcpp/src/runtime/denoiser.hpp index 48dd3b51d703..67aac538aa8c 100644 --- a/otherarch/sdcpp/src/runtime/denoiser.hpp +++ b/otherarch/sdcpp/src/runtime/denoiser.hpp @@ -306,8 +306,33 @@ struct KarrasScheduler : SigmaScheduler { }; struct BetaScheduler : SigmaScheduler { - static constexpr double alpha = 0.6; - static constexpr double beta = 0.6; + double alpha = 0.6; + double beta = 0.6; + + explicit BetaScheduler(const char* extra_sample_args = nullptr) { + parse_extra_sample_args(extra_sample_args); + LOG_DEBUG("Beta scheduler: alpha=%.4f, beta=%.4f", alpha, beta); + } + + void parse_extra_sample_args(const char* extra_sample_args) { + for (const auto& [key, value] : parse_key_value_args(extra_sample_args, "beta scheduler arg")) { + if (key == "alpha") { + float parsed; + if (!parse_strict_float(value, parsed) || parsed <= 0.0) { + LOG_WARN("ignoring invalid beta scheduler arg '%s=%s'", key.c_str(), value.c_str()); + } else { + alpha = static_cast(parsed); + } + } else if (key == "beta") { + float parsed; + if (!parse_strict_float(value, parsed) || parsed <= 0.0) { + LOG_WARN("ignoring invalid beta scheduler arg '%s=%s'", key.c_str(), value.c_str()); + } else { + beta = static_cast(parsed); + } + } + } + } static double log_beta(double a, double b) { return std::lgamma(a) + std::lgamma(b) - std::lgamma(a + b); @@ -1032,7 +1057,7 @@ struct Denoiser { break; case BETA_SCHEDULER: LOG_INFO("get_sigmas with Beta scheduler"); - scheduler = std::make_shared(); + scheduler = std::make_shared(extra_sample_args); break; case EXPONENTIAL_SCHEDULER: LOG_INFO("get_sigmas exponential scheduler"); diff --git a/otherarch/sdcpp/src/stable-diffusion.cpp b/otherarch/sdcpp/src/stable-diffusion.cpp index ef8b17b38066..857da289c2fe 100644 --- a/otherarch/sdcpp/src/stable-diffusion.cpp +++ b/otherarch/sdcpp/src/stable-diffusion.cpp @@ -780,8 +780,6 @@ class StableDiffusionGGML { } } - bool is_unet = sd_version_is_unet(model_loader.get_sd_version()); - // begin kcpp replacements SDVersion tempver = model_loader.get_sd_version(); bool fallback_swapped = false; @@ -1012,24 +1010,21 @@ class StableDiffusionGGML { if (strlen(SAFE_STR(sd_ctx_params->clip_l_path)) > 0) { LOG_INFO("loading clip_l from '%s'", sd_ctx_params->clip_l_path); - std::string prefix = is_unet ? "cond_stage_model.transformer." : "text_encoders.clip_l.transformer."; - if (!model_loader.init_from_file(sd_ctx_params->clip_l_path, prefix)) { + if (!model_loader.init_from_file(sd_ctx_params->clip_l_path, "clip_l.")) { LOG_WARN("loading clip_l from '%s' failed", sd_ctx_params->clip_l_path); } } if (strlen(SAFE_STR(sd_ctx_params->clip_g_path)) > 0) { LOG_INFO("loading clip_g from '%s'", sd_ctx_params->clip_g_path); - std::string prefix = is_unet ? "cond_stage_model.1.transformer." : "text_encoders.clip_g.transformer."; - if (!model_loader.init_from_file(sd_ctx_params->clip_g_path, prefix)) { + if (!model_loader.init_from_file(sd_ctx_params->clip_g_path, "clip_g.")) { LOG_WARN("loading clip_g from '%s' failed", sd_ctx_params->clip_g_path); } } if (strlen(SAFE_STR(sd_ctx_params->clip_vision_path)) > 0) { LOG_INFO("loading clip_vision from '%s'", sd_ctx_params->clip_vision_path); - std::string prefix = "cond_stage_model.transformer."; - if (!model_loader.init_from_file(sd_ctx_params->clip_vision_path, prefix)) { + if (!model_loader.init_from_file(sd_ctx_params->clip_vision_path, "clip_vision.")) { LOG_WARN("loading clip_vision from '%s' failed", sd_ctx_params->clip_vision_path); } } From a4dd6c118563f8de446b7213e8253ba46e41119c Mon Sep 17 00:00:00 2001 From: Wagner Bruna Date: Sun, 2 Aug 2026 14:42:49 -0300 Subject: [PATCH 4/6] sd: sync with master-810-db99efd --- otherarch/sdcpp/examples/common/common.cpp | 6 +- otherarch/sdcpp/include/stable-diffusion.h | 1 + .../sdcpp/src/model/adapter/ip_adapter.hpp | 153 ++++++++++++++++-- otherarch/sdcpp/src/model/adapter/lora.hpp | 112 ++++++++----- otherarch/sdcpp/src/model/diffusion/krea2.hpp | 27 +++- otherarch/sdcpp/src/name_conversion.cpp | 2 + otherarch/sdcpp/src/runtime/denoiser.hpp | 84 ++++++++++ otherarch/sdcpp/src/stable-diffusion.cpp | 131 ++++++++------- 8 files changed, 394 insertions(+), 122 deletions(-) diff --git a/otherarch/sdcpp/examples/common/common.cpp b/otherarch/sdcpp/examples/common/common.cpp index ceb7814308e2..66d1d3311ae8 100644 --- a/otherarch/sdcpp/examples/common/common.cpp +++ b/otherarch/sdcpp/examples/common/common.cpp @@ -1008,7 +1008,7 @@ ArgOptions SDGenerationParams::get_options() { &hires_upscaler}, {"", "--extra-sample-args", - "extra sampler/scheduler/guidance args, key=value list. CFG supports guidance_schedule; APG supports apg_eta, apg_momentum, apg_norm_threshold, apg_norm_threshold_smoothing; SLG supports slg_uncond; lcm supports noise_clip_std, noise_scale_start, noise_scale_end; flux supports base_shift, max_shift; ltx2 supports max_shift, base_shift, stretch, terminal; euler_ge supports gamma; beta scheduler supports alpha, beta; logit_normal supports mu, std, logsnr_min, logsnr_max, resolution_aware", + "extra sampler/scheduler/guidance args, key=value list. CFG supports guidance_schedule; APG supports apg_eta, apg_momentum, apg_norm_threshold, apg_norm_threshold_smoothing; SLG supports slg_uncond; lcm supports noise_clip_std, noise_scale_start, noise_scale_end; flux supports base_shift, max_shift; ltx2 supports max_shift, base_shift, stretch, terminal; euler_ge supports gamma; beta scheduler supports alpha, beta; logit_normal supports mu, std, logsnr_min, logsnr_max, resolution_aware; lms supports lms_divisions", (int)',', &extra_sample_args}, {"", @@ -1538,12 +1538,12 @@ ArgOptions SDGenerationParams::get_options() { on_seed_arg}, {"", "--sampling-method", - "sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, dpm++2m, dpm++2mv2, dpm++2m_sde, dpm++2m_sde_bt, ipndm, ipndm_v, lcm, ddim_trailing, tcd, res_multistep, res_2s, er_sde, euler_cfg_pp, euler_a_cfg_pp]" + "sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, dpm++2m, dpm++2mv2, dpm++2m_sde, dpm++2m_sde_bt, ipndm, ipndm_v, lcm, ddim_trailing, tcd, res_multistep, res_2s, er_sde, euler_cfg_pp, euler_a_cfg_pp, lms]" "(default: euler for Flux/SD3/Wan, euler_a otherwise)", on_sample_method_arg}, {"", "--high-noise-sampling-method", - "(high noise) sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, dpm++2m, dpm++2mv2, dpm++2m_sde, dpm++2m_sde_bt, ipndm, ipndm_v, lcm, ddim_trailing, tcd, res_multistep, res_2s, er_sde, euler_cfg_pp, euler_a_cfg_pp]" + "(high noise) sampling method, one of [euler, euler_a, heun, dpm2, dpm++2s_a, dpm++2m, dpm++2mv2, dpm++2m_sde, dpm++2m_sde_bt, ipndm, ipndm_v, lcm, ddim_trailing, tcd, res_multistep, res_2s, er_sde, euler_cfg_pp, euler_a_cfg_pp, lms]" " default: euler for Flux/SD3/Wan, euler_a otherwise", on_high_noise_sample_method_arg}, {"", diff --git a/otherarch/sdcpp/include/stable-diffusion.h b/otherarch/sdcpp/include/stable-diffusion.h index 60c99992f5f9..5af78704ee01 100644 --- a/otherarch/sdcpp/include/stable-diffusion.h +++ b/otherarch/sdcpp/include/stable-diffusion.h @@ -56,6 +56,7 @@ enum sample_method_t { EULER_GE_SAMPLE_METHOD, DPMPP2M_SDE_SAMPLE_METHOD, DPMPP2M_SDE_BT_SAMPLE_METHOD, + LMS_SAMPLE_METHOD, SAMPLE_METHOD_COUNT }; diff --git a/otherarch/sdcpp/src/model/adapter/ip_adapter.hpp b/otherarch/sdcpp/src/model/adapter/ip_adapter.hpp index 35fd1bb88ed5..cc4933e755b2 100644 --- a/otherarch/sdcpp/src/model/adapter/ip_adapter.hpp +++ b/otherarch/sdcpp/src/model/adapter/ip_adapter.hpp @@ -31,8 +31,92 @@ namespace IPAdapter { } }; + struct Resampler : public GGMLBlock { + int64_t dim = 1280; + int64_t depth = 4; + int64_t num_queries = 16; + int64_t embed_dim = 1280; + int64_t output_dim = 2048; + int64_t ff_inner = 5120; + int64_t dim_head = 64; + int64_t heads = 20; + + void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, const std::string prefix = "") override { + params["latents"] = ggml_new_tensor_3d(ctx, GGML_TYPE_F32, dim, num_queries, 1); + } + + Resampler() {} + Resampler(int64_t dim, int64_t depth, int64_t num_queries, int64_t embed_dim, int64_t output_dim, int64_t ff_inner) + : dim(dim), depth(depth), num_queries(num_queries), embed_dim(embed_dim), output_dim(output_dim), ff_inner(ff_inner) { + heads = dim / dim_head; + blocks["proj_in"] = std::shared_ptr(new Linear(embed_dim, dim, true)); + blocks["proj_out"] = std::shared_ptr(new Linear(dim, output_dim, true)); + blocks["norm_out"] = std::shared_ptr(new LayerNorm(output_dim)); + for (int64_t i = 0; i < depth; i++) { + std::string p = "layers." + std::to_string(i); + blocks[p + ".0.norm1"] = std::shared_ptr(new LayerNorm(dim)); + blocks[p + ".0.norm2"] = std::shared_ptr(new LayerNorm(dim)); + blocks[p + ".0.to_q"] = std::shared_ptr(new Linear(dim, dim, false)); + blocks[p + ".0.to_kv"] = std::shared_ptr(new Linear(dim, dim * 2, false)); + blocks[p + ".0.to_out"] = std::shared_ptr(new Linear(dim, dim, false)); + blocks[p + ".1.0"] = std::shared_ptr(new LayerNorm(dim)); + blocks[p + ".1.1"] = std::shared_ptr(new Linear(dim, ff_inner, false)); + blocks[p + ".1.3"] = std::shared_ptr(new Linear(ff_inner, dim, false)); + } + } + + ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* image_embeds) { + int64_t N = image_embeds->ne[2]; + auto proj_in = std::dynamic_pointer_cast(blocks["proj_in"]); + auto proj_out = std::dynamic_pointer_cast(blocks["proj_out"]); + auto norm_out = std::dynamic_pointer_cast(blocks["norm_out"]); + + ggml_tensor* x = proj_in->forward(ctx, image_embeds); + ggml_tensor* latents = params["latents"]; + if (N > 1) { + latents = ggml_repeat(ctx->ggml_ctx, latents, ggml_new_tensor_3d(ctx->ggml_ctx, GGML_TYPE_F32, dim, num_queries, N)); + } + + for (int64_t i = 0; i < depth; i++) { + std::string p = "layers." + std::to_string(i); + auto norm1 = std::dynamic_pointer_cast(blocks[p + ".0.norm1"]); + auto norm2 = std::dynamic_pointer_cast(blocks[p + ".0.norm2"]); + auto to_q = std::dynamic_pointer_cast(blocks[p + ".0.to_q"]); + auto to_kv = std::dynamic_pointer_cast(blocks[p + ".0.to_kv"]); + auto to_out = std::dynamic_pointer_cast(blocks[p + ".0.to_out"]); + + ggml_tensor* xn = norm1->forward(ctx, x); + ggml_tensor* ln = norm2->forward(ctx, latents); + ggml_tensor* q = to_q->forward(ctx, ln); + ggml_tensor* kv_in = ggml_concat(ctx->ggml_ctx, xn, ln, 1); + ggml_tensor* kv = to_kv->forward(ctx, kv_in); + int64_t L = kv->ne[1]; + ggml_tensor* k = ggml_cont(ctx->ggml_ctx, ggml_view_3d(ctx->ggml_ctx, kv, dim, L, N, kv->nb[1], kv->nb[2], 0)); + ggml_tensor* v = ggml_cont(ctx->ggml_ctx, ggml_view_3d(ctx->ggml_ctx, kv, dim, L, N, kv->nb[1], kv->nb[2], dim * kv->nb[0])); + ggml_tensor* attn = ggml_ext_attention_ext(ctx->ggml_ctx, ctx->backend, q, k, v, heads, nullptr, false, false); + attn = to_out->forward(ctx, attn); + latents = ggml_add(ctx->ggml_ctx, latents, attn); + + auto ff_norm = std::dynamic_pointer_cast(blocks[p + ".1.0"]); + auto ff_fc1 = std::dynamic_pointer_cast(blocks[p + ".1.1"]); + auto ff_fc2 = std::dynamic_pointer_cast(blocks[p + ".1.3"]); + ggml_tensor* h = ff_norm->forward(ctx, latents); + h = ff_fc1->forward(ctx, h); + h = ggml_gelu_erf(ctx->ggml_ctx, h); + h = ff_fc2->forward(ctx, h); + latents = ggml_add(ctx->ggml_ctx, latents, h); + } + + latents = proj_out->forward(ctx, latents); + latents = norm_out->forward(ctx, latents); + return latents; + } + }; + struct IPAdapterRunner : public GGMLRunner { ImageProjModel image_proj; + Resampler resampler; + bool is_plus = false; int64_t num_tokens = 4; std::string prefix; @@ -41,21 +125,54 @@ namespace IPAdapter { const std::string prefix, std::shared_ptr weight_manager = nullptr) : GGMLRunner(backend, weight_manager), prefix(prefix) { - int64_t ctx_dim = 768; - int64_t clip_dim = 1024; - int64_t out_dim = 3072; - auto norm_iter = tensor_storage_map.find(prefix + ".image_proj.norm.weight"); - if (norm_iter != tensor_storage_map.end()) { - ctx_dim = norm_iter->second.ne[0]; + is_plus = tensor_storage_map.find(prefix + ".image_proj.latents") != tensor_storage_map.end(); + if (is_plus) { + int64_t dim = 1280; + int64_t num_queries = 16; + int64_t embed_dim = 1280; + int64_t output_dim = 2048; + int64_t ff_inner = 5120; + auto latents_iter = tensor_storage_map.find(prefix + ".image_proj.latents"); + if (latents_iter != tensor_storage_map.end()) { + dim = latents_iter->second.ne[0]; + num_queries = latents_iter->second.ne[1]; + } + auto proj_in_iter = tensor_storage_map.find(prefix + ".image_proj.proj_in.weight"); + if (proj_in_iter != tensor_storage_map.end()) { + embed_dim = proj_in_iter->second.ne[0]; + } + auto proj_out_iter = tensor_storage_map.find(prefix + ".image_proj.proj_out.weight"); + if (proj_out_iter != tensor_storage_map.end()) { + output_dim = proj_out_iter->second.ne[1]; + } + auto ff_iter = tensor_storage_map.find(prefix + ".image_proj.layers.0.1.1.weight"); + if (ff_iter != tensor_storage_map.end()) { + ff_inner = ff_iter->second.ne[1]; + } + int64_t depth = 0; + while (tensor_storage_map.find(prefix + ".image_proj.layers." + std::to_string(depth) + ".0.to_q.weight") != tensor_storage_map.end()) { + depth++; + } + num_tokens = num_queries; + resampler = Resampler(dim, depth, num_queries, embed_dim, output_dim, ff_inner); + resampler.init(params_ctx, tensor_storage_map, prefix + ".image_proj"); + } else { + int64_t ctx_dim = 768; + int64_t clip_dim = 1024; + int64_t out_dim = 3072; + auto norm_iter = tensor_storage_map.find(prefix + ".image_proj.norm.weight"); + if (norm_iter != tensor_storage_map.end()) { + ctx_dim = norm_iter->second.ne[0]; + } + auto proj_iter = tensor_storage_map.find(prefix + ".image_proj.proj.weight"); + if (proj_iter != tensor_storage_map.end()) { + clip_dim = proj_iter->second.ne[0]; + out_dim = proj_iter->second.ne[1]; + } + num_tokens = out_dim / ctx_dim; + image_proj = ImageProjModel(num_tokens, ctx_dim, clip_dim); + image_proj.init(params_ctx, tensor_storage_map, prefix + ".image_proj"); } - auto proj_iter = tensor_storage_map.find(prefix + ".image_proj.proj.weight"); - if (proj_iter != tensor_storage_map.end()) { - clip_dim = proj_iter->second.ne[0]; - out_dim = proj_iter->second.ne[1]; - } - num_tokens = out_dim / ctx_dim; - image_proj = ImageProjModel(num_tokens, ctx_dim, clip_dim); - image_proj.init(params_ctx, tensor_storage_map, prefix + ".image_proj"); } std::string get_desc() override { @@ -63,14 +180,18 @@ namespace IPAdapter { } void get_param_tensors(std::map& tensors, const std::string = "") { - image_proj.get_param_tensors(tensors, prefix + ".image_proj"); + if (is_plus) { + resampler.get_param_tensors(tensors, prefix + ".image_proj"); + } else { + image_proj.get_param_tensors(tensors, prefix + ".image_proj"); + } } ggml_cgraph* build_graph(const sd::Tensor& image_embeds_tensor) { ggml_cgraph* gf = new_graph_custom(1024); ggml_tensor* embeds = make_input(image_embeds_tensor); auto runner_ctx = get_context(); - ggml_tensor* out = image_proj.forward(&runner_ctx, embeds); + ggml_tensor* out = is_plus ? resampler.forward(&runner_ctx, embeds) : image_proj.forward(&runner_ctx, embeds); ggml_build_forward_expand(gf, out); return gf; } diff --git a/otherarch/sdcpp/src/model/adapter/lora.hpp b/otherarch/sdcpp/src/model/adapter/lora.hpp index e91ca0bd1cd2..deef2c9b2aa4 100644 --- a/otherarch/sdcpp/src/model/adapter/lora.hpp +++ b/otherarch/sdcpp/src/model/adapter/lora.hpp @@ -588,6 +588,10 @@ struct LoraModel : public GGMLRunner { const std::string& model_tensor_name) { ggml_tensor* out_diff = nullptr; int index = 0; + + std::vector used_tensors; + bool is_conv2d = forward_params.op_type == WeightAdapter::ForwardParams::op_type_t::OP_CONV2D; + while (true) { std::string key; if (index == 0) { @@ -595,7 +599,6 @@ struct LoraModel : public GGMLRunner { } else { key = model_tensor_name + "." + std::to_string(index); } - bool is_conv2d = forward_params.op_type == WeightAdapter::ForwardParams::op_type_t::OP_CONV2D; std::string lokr_w1_name = "lora." + key + ".lokr_w1"; std::string lokr_w1_a_name = "lora." + key + ".lokr_w1_a"; @@ -663,7 +666,6 @@ struct LoraModel : public GGMLRunner { if (iter != lora_tensors.end()) { float alpha = ggml_ext_backend_tensor_get_f32(iter->second); scale_value = alpha / rank; - applied_lora_tensors.insert(alpha_name); } if (rank == 1) { @@ -678,19 +680,27 @@ struct LoraModel : public GGMLRunner { out_diff = ggml_concat(ctx, out_diff, curr_out_diff, 0); } - if (lokr_w1) - applied_lora_tensors.insert(lokr_w1_name); - if (lokr_w1_a) - applied_lora_tensors.insert(lokr_w1_a_name); - if (lokr_w1_b) - applied_lora_tensors.insert(lokr_w1_b_name); - if (lokr_w2) - applied_lora_tensors.insert(lokr_w2_name); - if (lokr_w2_a) - applied_lora_tensors.insert(lokr_w2_a_name); - if (lokr_w2_b) - applied_lora_tensors.insert(lokr_w2_b_name); - applied_lora_tensors.insert(alpha_name); + if (lokr_w1) { + used_tensors.push_back(lokr_w1_name); + } + if (lokr_w1_a) { + used_tensors.push_back(lokr_w1_a_name); + } + if (lokr_w1_b) { + used_tensors.push_back(lokr_w1_b_name); + } + if (lokr_w2) { + used_tensors.push_back(lokr_w2_name); + } + if (lokr_w2_a) { + used_tensors.push_back(lokr_w2_a_name); + } + if (lokr_w2_b) { + used_tensors.push_back(lokr_w2_b_name); + } + if (iter != lora_tensors.end()) { + used_tensors.push_back(alpha_name); + } index++; continue; @@ -740,10 +750,8 @@ struct LoraModel : public GGMLRunner { const int64_t down_in = lora_down->ne[0]; const int64_t down_out = lora_down->ne[1]; const int64_t up_in = lora_up->ne[0]; - const int64_t up_out = lora_up->ne[1]; - bool compatible = down_in == model_weight->ne[0] && - up_out == model_weight->ne[1]; + bool compatible = down_in == model_weight->ne[0]; if (lora_mid != nullptr) { compatible = compatible && lora_mid->ne[0] == down_out && @@ -755,45 +763,43 @@ struct LoraModel : public GGMLRunner { if (!compatible) { skipped_incompatible_lora_tensors.insert(lora_down_name); skipped_incompatible_lora_tensors.insert(lora_up_name); - skipped_incompatible_lora_tensors.insert(lora_mid_name); - skipped_incompatible_lora_tensors.insert(scale_name); - skipped_incompatible_lora_tensors.insert(alpha_name); + if (lora_mid != nullptr) { + skipped_incompatible_lora_tensors.insert(lora_mid_name); + } + if (lora_tensors.find(scale_name) != lora_tensors.end()) { + skipped_incompatible_lora_tensors.insert(scale_name); + } else if (lora_tensors.find(alpha_name) != lora_tensors.end()) { + skipped_incompatible_lora_tensors.insert(alpha_name); + } if (warned_incompatible_model_tensors.insert(model_tensor_name).second) { - LOG_WARN("skip incompatible LoRA tensor |%s|: model shape = [%lld, %lld], down shape = [%lld, %lld], up shape = [%lld, %lld]", + LOG_WARN("skip incompatible LoRA tensor |%s|: model input dim = %lld, down shape = [%lld, %lld], up shape = [%lld, %lld]", model_tensor_name.c_str(), static_cast(model_weight->ne[0]), - static_cast(model_weight->ne[1]), static_cast(down_in), static_cast(down_out), static_cast(up_in), - static_cast(up_out)); + static_cast(lora_up->ne[1])); } index++; continue; } } - applied_lora_tensors.insert(lora_up_name); - applied_lora_tensors.insert(lora_down_name); - - if (lora_mid) { - applied_lora_tensors.insert(lora_mid_name); - } - float scale_value = 1.0f; + std::string scale_tensor_name; int64_t rank = lora_down->ne[ggml_n_dims(lora_down) - 1]; iter = lora_tensors.find(scale_name); if (iter != lora_tensors.end()) { - scale_value = ggml_ext_backend_tensor_get_f32(iter->second); - applied_lora_tensors.insert(scale_name); + scale_value = ggml_ext_backend_tensor_get_f32(iter->second); + scale_tensor_name = scale_name; } else { iter = lora_tensors.find(alpha_name); if (iter != lora_tensors.end()) { - float alpha = ggml_ext_backend_tensor_get_f32(iter->second); - scale_value = alpha / rank; + float alpha = ggml_ext_backend_tensor_get_f32(iter->second); + scale_value = alpha / rank; + scale_tensor_name = alpha_name; // LOG_DEBUG("rank %s %ld %.2f %.2f", alpha_name.c_str(), rank, alpha, scale_value); - applied_lora_tensors.insert(alpha_name); } } scale_value *= multiplier; @@ -853,15 +859,45 @@ struct LoraModel : public GGMLRunner { } auto curr_out_diff = ggml_ext_scale(ctx, lx, scale_value, true); - if (out_diff == nullptr) { out_diff = curr_out_diff; } else { - out_diff = ggml_concat(ctx, out_diff, curr_out_diff, 0); + out_diff = ggml_concat(ctx, out_diff, curr_out_diff, is_conv2d ? 2 : 0); + } + + used_tensors.push_back(lora_up_name); + used_tensors.push_back(lora_down_name); + if (lora_mid) { + used_tensors.push_back(lora_mid_name); + } + if (!scale_tensor_name.empty()) { + used_tensors.push_back(scale_tensor_name); } index++; } + + if (out_diff == nullptr) + return nullptr; + + int64_t expected_out_dim = is_conv2d ? model_weight->ne[3] : model_weight->ne[1]; + int64_t actual_out_dim = out_diff->ne[is_conv2d ? 2 : 0]; + + if (actual_out_dim != expected_out_dim) { + for (const auto& name : used_tensors) { + skipped_incompatible_lora_tensors.insert(name); + } + if (warned_incompatible_model_tensors.insert(model_tensor_name).second) { + LOG_WARN("skip incompatible LoRA tensors for |%s|: output dim %lld != model dim %lld", + model_tensor_name.c_str(), actual_out_dim, expected_out_dim); + } + return nullptr; + } + + for (const auto& name : used_tensors) { + applied_lora_tensors.insert(name); + } + return out_diff; } diff --git a/otherarch/sdcpp/src/model/diffusion/krea2.hpp b/otherarch/sdcpp/src/model/diffusion/krea2.hpp index 9121069ad507..b3947b71d933 100644 --- a/otherarch/sdcpp/src/model/diffusion/krea2.hpp +++ b/otherarch/sdcpp/src/model/diffusion/krea2.hpp @@ -180,9 +180,12 @@ namespace Krea2 { ggml_tensor* forward(GGMLRunnerContext* ctx, ggml_tensor* x) override { ggml_tensor* scale = params["scale"]; - scale = ggml_add(ctx->ggml_ctx, scale, ggml_ext_ones(ctx->ggml_ctx, scale->ne[0], 1, 1, 1)); - x = ggml_rms_norm(ctx->ggml_ctx, x, eps); - x = ggml_mul_inplace(ctx->ggml_ctx, x, scale); + if (ctx->weight_adapter) { + scale = ctx->weight_adapter->patch_weight(ctx->ggml_ctx, ctx->backend, scale, prefix + "scale.weight"); + } + scale = ggml_add(ctx->ggml_ctx, scale, ggml_ext_ones(ctx->ggml_ctx, scale->ne[0], 1, 1, 1)); + x = ggml_rms_norm(ctx->ggml_ctx, x, eps); + x = ggml_mul_inplace(ctx->ggml_ctx, x, scale); return x; } }; @@ -295,10 +298,11 @@ namespace Krea2 { class KreaDoubleSharedModulation : public GGMLBlock { protected: int64_t dim; + std::string prefix; void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, const std::string prefix = "") override { GGML_UNUSED(tensor_storage_map); - GGML_UNUSED(prefix); + this->prefix = prefix; params["lin"] = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, dim * 6); } @@ -307,7 +311,11 @@ namespace Krea2 { : dim(dim) {} std::vector forward(GGMLRunnerContext* ctx, ggml_tensor* vec) { - auto lin = ggml_repeat(ctx->ggml_ctx, params["lin"], vec); + auto lin = params["lin"]; + if (ctx->weight_adapter) { + lin = ctx->weight_adapter->patch_weight(ctx->ggml_ctx, ctx->backend, lin, prefix + "lin.weight"); + } + lin = ggml_repeat(ctx->ggml_ctx, lin, vec); auto out = ggml_add(ctx->ggml_ctx, vec, lin); return ggml_ext_chunk(ctx->ggml_ctx, out, 6, 0); } @@ -316,10 +324,11 @@ namespace Krea2 { class KreaFinalModulation : public GGMLBlock { protected: int64_t dim; + std::string prefix; void init_params(ggml_context* ctx, const String2TensorStorage& tensor_storage_map = {}, const std::string prefix = "") override { GGML_UNUSED(tensor_storage_map); - GGML_UNUSED(prefix); + this->prefix = prefix; params["lin"] = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, dim, 2); } @@ -328,7 +337,11 @@ namespace Krea2 { : dim(dim) {} std::vector forward(GGMLRunnerContext* ctx, ggml_tensor* vec) { - auto out = ggml_add(ctx->ggml_ctx, params["lin"], vec); + auto lin = params["lin"]; + if (ctx->weight_adapter) { + lin = ctx->weight_adapter->patch_weight(ctx->ggml_ctx, ctx->backend, lin, prefix + "lin.weight"); + } + auto out = ggml_add(ctx->ggml_ctx, lin, vec); return ggml_ext_chunk(ctx->ggml_ctx, out, 2, 1); } }; diff --git a/otherarch/sdcpp/src/name_conversion.cpp b/otherarch/sdcpp/src/name_conversion.cpp index af8a7f572d1f..d6e296e15dd1 100644 --- a/otherarch/sdcpp/src/name_conversion.cpp +++ b/otherarch/sdcpp/src/name_conversion.cpp @@ -1384,6 +1384,8 @@ std::string convert_tensor_name(std::string name, SDVersion version) { {".lora_B.weight", ".weight.lora_up"}, {".lora_A.default.weight", ".weight.lora_down"}, {".lora_B.default.weight", ".weight.lora_up"}, + {".lora_A", ".weight.lora_down"}, + {".lora_B", ".weight.lora_up"}, {".lora_linear", ".weight.alpha"}, {".alpha", ".weight.alpha"}, {".scale", ".weight.scale"}, diff --git a/otherarch/sdcpp/src/runtime/denoiser.hpp b/otherarch/sdcpp/src/runtime/denoiser.hpp index 67aac538aa8c..f58f42b55fcd 100644 --- a/otherarch/sdcpp/src/runtime/denoiser.hpp +++ b/otherarch/sdcpp/src/runtime/denoiser.hpp @@ -2578,6 +2578,88 @@ static sd::Tensor sample_tcd(denoise_cb_t model, return x; } +static sd::Tensor sample_lms(denoise_cb_t model, + sd::Tensor x, + const std::vector& sigmas, + const SamplerExtraArgs& extra_sample_args) { + // Linear Multi-Step from https://github.com/crowsonkb/k-diffusion + + int divisions = 1000; + for (const auto& [key, value] : extra_sample_args) { + int parsed = 0; + if (key == "lms_divisions") { + if (!parse_strict_int(value, parsed)) { + LOG_WARN("ignoring invalid lms extra sample arg '%s=%s'", key.c_str(), value.c_str()); + continue; + } + divisions = parsed; // std::max(1, parsed); + // values above 35M produce noise, can be fixed by double precision + // values < 1 always produce noise + } + } + LOG_DEBUG("linear multi-step sampler: integrating using %i division%s", divisions, (divisions == 1) ? "" : "s"); + + auto linear_multistep_coeff = [=](const int order, const int m, const int j) -> float { + if (!divisions) + return sigmas[m + 1] - sigmas[m]; // delta / 0 * 0 +#define LMS_PRECISION float // double + const LMS_PRECISION a = sigmas[m], dx = (sigmas[m + 1] - a) / divisions, s = sigmas[m - j]; + const LMS_PRECISION b0 = a + 0.5f * dx; // using Riemann middle integral + LMS_PRECISION sum = 0.0f; + for (int h = 0; h < divisions; h++) { + const LMS_PRECISION b = h * dx + b0; + LMS_PRECISION prod = 1.0f; + for (int k = 0; k < j; k++) { + const LMS_PRECISION t = sigmas[m - k]; + prod *= (b - t) / (s - t); + } + for (int k = j + 1; k < order; k++) { + const LMS_PRECISION t = sigmas[m - k]; + prod *= (b - t) / (s - t); + } + sum += prod; + } + return sum * dx; + }; + + const int max_order = 4; + float lms_coeff[max_order]; + std::vector> hist = {}; + + int steps = static_cast(sigmas.size()) - 1; + for (int i = 0; i < steps; i++) { + const float sigma = sigmas[i]; + + auto denoised_opt = model(x, sigma, i + 1); + if (denoised_opt.pred.empty()) { + return {}; + } + sd::Tensor denoised = std::move(denoised_opt.pred); + + const int order = std::min(max_order, i + 1); + for (int c = 0; c < order; c++) // computing coefficients + lms_coeff[c] = linear_multistep_coeff(order, i, c); + + sd::Tensor d_cur = (x - denoised) / sigma; + switch (order) { + case 4: // derivative + 3 history points + x += hist[hist.size() - 2] * lms_coeff[3]; + case 3: + x += hist[hist.size() - 1] * lms_coeff[2]; + case 2: + x += hist.back() * lms_coeff[1]; + case 1: + x += d_cur * lms_coeff[0]; + } + + if (hist.size() == static_cast(max_order - 1)) { + hist.erase(hist.begin()); + } + hist.push_back(std::move(d_cur)); + } + return x; +} + static sd::Tensor sample_euler_cfg_pp(denoise_cb_t model, sd::Tensor x, const std::vector& sigmas) { @@ -2739,6 +2821,8 @@ static sd::Tensor sample_k_diffusion(sample_method_t method, return sample_euler_ancestral(model, std::move(x), sigmas, rng, is_flow_denoiser, eta); case TCD_SAMPLE_METHOD: return sample_tcd(model, std::move(x), sigmas, rng, eta); + case LMS_SAMPLE_METHOD: + return sample_lms(model, std::move(x), sigmas, extra_args); case EULER_CFG_PP_SAMPLE_METHOD: return sample_euler_cfg_pp(model, std::move(x), sigmas); case EULER_A_CFG_PP_SAMPLE_METHOD: diff --git a/otherarch/sdcpp/src/stable-diffusion.cpp b/otherarch/sdcpp/src/stable-diffusion.cpp index 857da289c2fe..036aa3233d6b 100644 --- a/otherarch/sdcpp/src/stable-diffusion.cpp +++ b/otherarch/sdcpp/src/stable-diffusion.cpp @@ -145,6 +145,7 @@ const char* sampling_methods_str[] = { "Euler GE", "DPM++ (2M) SDE", "DPM++ (2M) SDE BT", + "LMS", }; /*================================================== Helper Functions ================================================*/ @@ -699,42 +700,15 @@ class StableDiffusionGGML { LOG_DEBUG("loaded alphas_cumprod from model file"); } - bool init(const sd_ctx_params_t* sd_ctx_params_kcpp) { + bool init_model_loader(ModelLoader& model_loader, + const sd_ctx_params_t* sd_ctx_params_kcpp, + bool& use_tae, + bool& use_audio_vae, + bool& use_control_net) { + // kcpp make sd_ctx_params mutable sd_ctx_params_t sd_ctx_params_local = *sd_ctx_params_kcpp; sd_ctx_params_t *sd_ctx_params = &sd_ctx_params_local; - n_threads = sd_ctx_params->n_threads; - enable_mmap = sd_ctx_params->enable_mmap; - stream_layers = sd_ctx_params->stream_layers; - eager_load = sd_ctx_params->eager_load; - backend_spec = SAFE_STR(sd_ctx_params->backend); - params_backend_spec = SAFE_STR(sd_ctx_params->params_backend); - split_mode_spec = SAFE_STR(sd_ctx_params->split_mode); - auto_fit_enabled = sd_ctx_params->auto_fit; - max_vram_assignment.reset(0.f); - { - std::string error; - if (!max_vram_assignment.parse(SAFE_STR(sd_ctx_params->max_vram), &error)) { - LOG_ERROR("%s", error.c_str()); - return false; - } - } - - std::string rpc_servers_spec = SAFE_STR(sd_ctx_params->rpc_servers); - add_rpc_devices(rpc_servers_spec); - - bool use_tae = false; - bool use_audio_vae = false; - bool use_control_net = false; - - rng = get_rng(sd_ctx_params->rng_type); - if (sd_ctx_params->sampler_rng_type != RNG_TYPE_COUNT && sd_ctx_params->sampler_rng_type != sd_ctx_params->rng_type) { - sampler_rng = get_rng(sd_ctx_params->sampler_rng_type); - } else { - sampler_rng = rng; - } - - ggml_log_set(ggml_log_callback_default, nullptr); std::string clip_vision_fixed = SAFE_STR(sd_ctx_params->clip_vision_path); std::string clipg_path_fixed = SAFE_STR(sd_ctx_params->clip_g_path); @@ -747,11 +721,6 @@ class StableDiffusionGGML { std::string vae_path_fixed = SAFE_STR(sd_ctx_params->vae_path); std::string uncond_fixed = SAFE_STR(sd_ctx_params->uncond_diffusion_model_path); - model_manager = std::make_shared(); - model_manager->set_n_threads(n_threads); - model_manager->set_enable_mmap(enable_mmap); - ModelLoader& model_loader = model_manager->loader(); - if (strlen(SAFE_STR(sd_ctx_params->model_path)) > 0) { LOG_INFO("loading model from '%s'", sd_ctx_params->model_path); if (!model_loader.init_from_file(sd_ctx_params->model_path)) { @@ -1119,19 +1088,10 @@ class StableDiffusionGGML { model_loader.convert_tensors_name(); - version = model_loader.get_sd_version(); - if (version == VERSION_COUNT) { - LOG_ERROR("get sd version from file failed: '%s'", SAFE_STR(sd_ctx_params->model_path)); - return false; - } - - auto& tensor_storage_map = model_loader.get_tensor_storage_map(); - - LOG_INFO("Version: %s ", model_version_to_str[version]); ggml_type wtype = sd_type_to_ggml_type(sd_ctx_params->wtype); std::string tensor_type_rules = SAFE_STR(sd_ctx_params->tensor_type_rules); //kcpp: patch hidream to fix broken images on vulkan https://github.com/leejet/stable-diffusion.cpp/issues/1496 - if(version == VERSION_HIDREAM_O1 && tensor_type_rules.size()==0) + if(tempver == VERSION_HIDREAM_O1 && tensor_type_rules.size()==0) { tensor_type_rules = "^model.language_model.layers.[0-9]+.mlp.down_proj.weight=bf16"; } @@ -1139,9 +1099,63 @@ class StableDiffusionGGML { model_loader.set_wtype_override(wtype, tensor_type_rules); } + return true; + } + + bool init(const sd_ctx_params_t* sd_ctx_params) { + n_threads = sd_ctx_params->n_threads; + enable_mmap = sd_ctx_params->enable_mmap; + stream_layers = sd_ctx_params->stream_layers; + eager_load = sd_ctx_params->eager_load; + backend_spec = SAFE_STR(sd_ctx_params->backend); + params_backend_spec = SAFE_STR(sd_ctx_params->params_backend); + split_mode_spec = SAFE_STR(sd_ctx_params->split_mode); + auto_fit_enabled = sd_ctx_params->auto_fit; + max_vram_assignment.reset(0.f); + { + std::string error; + if (!max_vram_assignment.parse(SAFE_STR(sd_ctx_params->max_vram), &error)) { + LOG_ERROR("%s", error.c_str()); + return false; + } + } + + std::string rpc_servers_spec = SAFE_STR(sd_ctx_params->rpc_servers); + add_rpc_devices(rpc_servers_spec); + + bool use_tae = false; + bool use_audio_vae = false; + bool use_control_net = false; + + rng = get_rng(sd_ctx_params->rng_type); + if (sd_ctx_params->sampler_rng_type != RNG_TYPE_COUNT && sd_ctx_params->sampler_rng_type != sd_ctx_params->rng_type) { + sampler_rng = get_rng(sd_ctx_params->sampler_rng_type); + } else { + sampler_rng = rng; + } + + ggml_log_set(ggml_log_callback_default, nullptr); + + model_manager = std::make_shared(); + model_manager->set_n_threads(n_threads); + model_manager->set_enable_mmap(enable_mmap); + ModelLoader& model_loader = model_manager->loader(); + + if (!init_model_loader(model_loader, sd_ctx_params, use_tae, use_audio_vae, use_control_net)) { + return false; + } + + version = model_loader.get_sd_version(); + if (version == VERSION_COUNT) { + LOG_ERROR("get sd version from file failed: '%s'", SAFE_STR(sd_ctx_params->model_path)); + return false; + } else { + LOG_INFO("Version: %s ", model_version_to_str[version]); + } + if (auto_fit_enabled) { if (!sd::backend_fit::derive_backend_specs(model_loader, - wtype, + sd_type_to_ggml_type(sd_ctx_params->wtype), max_vram_assignment, backend_spec, params_backend_spec)) { @@ -1196,14 +1210,10 @@ class StableDiffusionGGML { if (sd_ctx_params->lora_apply_mode == LORA_APPLY_AUTO) { bool have_quantized_weight = false; - if (wtype != GGML_TYPE_COUNT && ggml_is_quantized(wtype)) { - have_quantized_weight = true; - } else { - for (const auto& [type, _] : wtype_stat) { - if (ggml_is_quantized(type)) { - have_quantized_weight = true; - break; - } + for (const auto& [type, _] : wtype_stat) { + if (ggml_is_quantized(type)) { + have_quantized_weight = true; + break; } } // Avoid full-model LoRA merge buffers on constrained setups. @@ -1247,6 +1257,8 @@ class StableDiffusionGGML { use_tae = true; } + auto& tensor_storage_map = model_loader.get_tensor_storage_map(); + { if (!ensure_backend_pair(SDBackendModule::TE) || !ensure_backend_pair(SDBackendModule::DIFFUSION)) { @@ -2396,7 +2408,9 @@ class StableDiffusionGGML { return; } auto image_tensor = sd_image_to_tensor(image); - auto embed = get_clip_vision_output(image_tensor, true, -1); + auto embed = ip_adapter->is_plus + ? get_clip_vision_output(image_tensor, false, 2) + : get_clip_vision_output(image_tensor, true, -1); if (embed.empty()) { return; } @@ -3476,6 +3490,7 @@ const char* sample_method_to_str[] = { "euler_ge", "dpm++2m_sde", "dpm++2m_sde_bt", + "lms", }; const char* sd_sample_method_name(enum sample_method_t sample_method) { From 477a8cfbe7f9ea5b51df4cc48fad2c55dd38e69c Mon Sep 17 00:00:00 2001 From: Wagner Bruna Date: Tue, 4 Aug 2026 21:11:53 -0300 Subject: [PATCH 5/6] sd: sync with master-812-ea7f0c8 --- Makefile | 2 +- otherarch/sdcpp/examples/cli/main.cpp | 43 + otherarch/sdcpp/examples/common/common.cpp | 86 +- otherarch/sdcpp/examples/common/common.h | 9 + otherarch/sdcpp/examples/common/media_io.cpp | 129 ++ otherarch/sdcpp/examples/common/media_io.h | 5 + .../sdcpp/examples/common/resource_owners.hpp | 31 + otherarch/sdcpp/include/stable-diffusion.h | 13 + .../sdcpp/src/conditioning/conditioner.hpp | 187 ++- otherarch/sdcpp/src/model.h | 6 + .../sdcpp/src/model/diffusion/minimax_h3.hpp | 1178 +++++++++++++++++ otherarch/sdcpp/src/model/diffusion/model.hpp | 24 + otherarch/sdcpp/src/model/te/llm.hpp | 395 +++++- otherarch/sdcpp/src/model/vae/audio_vae.hpp | 28 + .../sdcpp/src/model/vae/ltx_audio_vae.hpp | 17 +- .../src/model/vae/minimax_h3_audio_vae.hpp | 497 +++++++ .../sdcpp/src/model/vae/minimax_h3_vae.hpp | 805 +++++++++++ otherarch/sdcpp/src/model/vae/vae.hpp | 29 +- otherarch/sdcpp/src/model_loader.cpp | 4 + otherarch/sdcpp/src/name_conversion.cpp | 4 +- otherarch/sdcpp/src/stable-diffusion.cpp | 471 ++++++- 21 files changed, 3868 insertions(+), 95 deletions(-) create mode 100644 otherarch/sdcpp/src/model/diffusion/minimax_h3.hpp create mode 100644 otherarch/sdcpp/src/model/vae/audio_vae.hpp create mode 100644 otherarch/sdcpp/src/model/vae/minimax_h3_audio_vae.hpp create mode 100644 otherarch/sdcpp/src/model/vae/minimax_h3_vae.hpp diff --git a/Makefile b/Makefile index 64e6684a4da3..351ea6b39c0c 100644 --- a/Makefile +++ b/Makefile @@ -688,7 +688,7 @@ budget.o: common/reasoning-budget.cpp common/reasoning-budget.h chat.o: common/chat.cpp common/chat.h $(CXX) $(CXXFLAGS) -c $< -o $@ -SDCPP_COMMON_BASENAMES := include/stable-diffusion.h src/conditioning/conditioner.hpp src/core/backend_fit.cpp src/core/backend_fit.h src/core/ggml_extend_backend.cpp src/core/ggml_extend_backend.h src/core/ggml_extend.hpp src/core/ggml_graph_cut.cpp src/core/ggml_graph_cut.h src/core/layer_split_partition.cpp src/core/layer_split_partition.h src/core/ordered_map.hpp src/core/rng.hpp src/core/rng_mt19937.hpp src/core/rng_philox.hpp src/core/tensor_ggml.hpp src/core/tensor.hpp src/core/util.cpp src/core/util.h src/detailer.cpp src/detailer.h src/extensions/generation_extension.h src/extensions/photomaker_extension.cpp src/extensions/pulid_extension.cpp src/kcpp_sd_extensions.h src/model/adapter/ip_adapter.hpp src/model/adapter/lora.hpp src/model/adapter/pmid.hpp src/model/adapter/pulid.hpp src/model/common/block.hpp src/model/common/rope.hpp src/model/detector/yolov8.h src/model/diffusion/anima.hpp src/model/diffusion/animatediff.hpp src/model/diffusion/boogu.hpp src/model/diffusion/control.hpp src/model/diffusion/dit.hpp src/model/diffusion/ernie_image.hpp src/model/diffusion/flux.hpp src/model/diffusion/hidream_o1.hpp src/model/diffusion/hunyuan.hpp src/model/diffusion/ideogram4.hpp src/model/diffusion/krea2.hpp src/model/diffusion/lens.hpp src/model/diffusion/lingbot_video.hpp src/model/diffusion/ltxv.hpp src/model/diffusion/mage_flow.hpp src/model/diffusion/minit2i.hpp src/model/diffusion/mmdit.hpp src/model/diffusion/model.hpp src/model/diffusion/pid.hpp src/model/diffusion/qwen_image.hpp src/model/diffusion/sefi_image.hpp src/model/diffusion/unet.hpp src/model/diffusion/wan.hpp src/model/diffusion/z_image.hpp src/model.h src/model_io/binary_io.h src/model_io/gguf_io.cpp src/model_io/gguf_io.h src/model_io/gguf_reader_ext.h src/model_io/kcpp_sdcpp_quantized_safetensors.hpp src/model_io/pickle_io.cpp src/model_io/pickle_io.h src/model_io/safetensors_io.cpp src/model_io/safetensors_io.h src/model_io/streaming_writer.h src/model_io/tensor_storage.h src/model_io/torch_legacy_io.cpp src/model_io/torch_legacy_io.h src/model_io/torch_zip_io.cpp src/model_io/torch_zip_io.h src/model_loader.cpp src/model_loader.h src/model_manager.cpp src/model_manager.h src/model/te/clip.hpp src/model/te/llm.hpp src/model/te/t5.hpp src/model/upscaler/esrgan.hpp src/model/upscaler/ltx_latent_upscaler.hpp src/model/vae/auto_encoder_kl.hpp src/model/vae/hunyuan_vae.hpp src/model/vae/ltx_audio_vae.hpp src/model/vae/ltx_vae.hpp src/model/vae/mage_vae.hpp src/model/vae/tae.hpp src/model/vae/vae.hpp src/model/vae/wan_vae.hpp src/name_conversion.cpp src/name_conversion.h src/runtime/cache_dit.hpp src/runtime/condition_cache_utils.hpp src/runtime/denoiser.hpp src/runtime/easycache.hpp src/runtime/gits_noise.h src/runtime/guidance.cpp src/runtime/guidance.h src/runtime/imatrix.cpp src/runtime/imatrix.h src/runtime/latent-preview.h src/runtime/preprocessing.hpp src/runtime/sample-cache.cpp src/runtime/sample-cache.h src/runtime/spectrum.hpp src/runtime/ucache.hpp src/stable-diffusion.cpp src/tokenizers/bpe_tokenizer.cpp src/tokenizers/bpe_tokenizer.h src/tokenizers/clip_tokenizer.cpp src/tokenizers/clip_tokenizer.h src/tokenizers/gemma_tokenizer.cpp src/tokenizers/gemma_tokenizer.h src/tokenizers/gpt_oss_tokenizer.cpp src/tokenizers/gpt_oss_tokenizer.h src/tokenizers/mistral_tokenizer.cpp src/tokenizers/mistral_tokenizer.h src/tokenizers/qwen2_tokenizer.cpp src/tokenizers/qwen2_tokenizer.h src/tokenizers/t5_unigram_tokenizer.cpp src/tokenizers/t5_unigram_tokenizer.h src/tokenizers/tokenizer.cpp src/tokenizers/tokenizer.h src/tokenizers/tokenize_util.cpp src/tokenizers/tokenize_util.h src/tokenizers/vocab/vocab.h src/upscaler.cpp src/upscaler.h src/weight_manager.h +SDCPP_COMMON_BASENAMES := include/stable-diffusion.h src/conditioning/conditioner.hpp src/core/backend_fit.cpp src/core/backend_fit.h src/core/ggml_extend_backend.cpp src/core/ggml_extend_backend.h src/core/ggml_extend.hpp src/core/ggml_graph_cut.cpp src/core/ggml_graph_cut.h src/core/layer_split_partition.cpp src/core/layer_split_partition.h src/core/ordered_map.hpp src/core/rng.hpp src/core/rng_mt19937.hpp src/core/rng_philox.hpp src/core/tensor_ggml.hpp src/core/tensor.hpp src/core/util.cpp src/core/util.h src/detailer.cpp src/detailer.h src/extensions/generation_extension.h src/extensions/photomaker_extension.cpp src/extensions/pulid_extension.cpp src/kcpp_sd_extensions.h src/model/adapter/ip_adapter.hpp src/model/adapter/lora.hpp src/model/adapter/pmid.hpp src/model/adapter/pulid.hpp src/model/common/block.hpp src/model/common/rope.hpp src/model/detector/yolov8.h src/model/diffusion/anima.hpp src/model/diffusion/animatediff.hpp src/model/diffusion/boogu.hpp src/model/diffusion/control.hpp src/model/diffusion/dit.hpp src/model/diffusion/ernie_image.hpp src/model/diffusion/flux.hpp src/model/diffusion/hidream_o1.hpp src/model/diffusion/hunyuan.hpp src/model/diffusion/ideogram4.hpp src/model/diffusion/krea2.hpp src/model/diffusion/lens.hpp src/model/diffusion/lingbot_video.hpp src/model/diffusion/ltxv.hpp src/model/diffusion/mage_flow.hpp src/model/diffusion/minimax_h3.hpp src/model/diffusion/minit2i.hpp src/model/diffusion/mmdit.hpp src/model/diffusion/model.hpp src/model/diffusion/pid.hpp src/model/diffusion/qwen_image.hpp src/model/diffusion/sefi_image.hpp src/model/diffusion/unet.hpp src/model/diffusion/wan.hpp src/model/diffusion/z_image.hpp src/model.h src/model_io/binary_io.h src/model_io/gguf_io.cpp src/model_io/gguf_io.h src/model_io/gguf_reader_ext.h src/model_io/kcpp_sdcpp_quantized_safetensors.hpp src/model_io/pickle_io.cpp src/model_io/pickle_io.h src/model_io/safetensors_io.cpp src/model_io/safetensors_io.h src/model_io/streaming_writer.h src/model_io/tensor_storage.h src/model_io/torch_legacy_io.cpp src/model_io/torch_legacy_io.h src/model_io/torch_zip_io.cpp src/model_io/torch_zip_io.h src/model_loader.cpp src/model_loader.h src/model_manager.cpp src/model_manager.h src/model/te/clip.hpp src/model/te/llm.hpp src/model/te/t5.hpp src/model/upscaler/esrgan.hpp src/model/upscaler/ltx_latent_upscaler.hpp src/model/vae/audio_vae.hpp src/model/vae/auto_encoder_kl.hpp src/model/vae/hunyuan_vae.hpp src/model/vae/ltx_audio_vae.hpp src/model/vae/ltx_vae.hpp src/model/vae/mage_vae.hpp src/model/vae/minimax_h3_audio_vae.hpp src/model/vae/minimax_h3_vae.hpp src/model/vae/tae.hpp src/model/vae/vae.hpp src/model/vae/wan_vae.hpp src/name_conversion.cpp src/name_conversion.h src/runtime/cache_dit.hpp src/runtime/condition_cache_utils.hpp src/runtime/denoiser.hpp src/runtime/easycache.hpp src/runtime/gits_noise.h src/runtime/guidance.cpp src/runtime/guidance.h src/runtime/imatrix.cpp src/runtime/imatrix.h src/runtime/latent-preview.h src/runtime/preprocessing.hpp src/runtime/sample-cache.cpp src/runtime/sample-cache.h src/runtime/spectrum.hpp src/runtime/ucache.hpp src/stable-diffusion.cpp src/tokenizers/bpe_tokenizer.cpp src/tokenizers/bpe_tokenizer.h src/tokenizers/clip_tokenizer.cpp src/tokenizers/clip_tokenizer.h src/tokenizers/gemma_tokenizer.cpp src/tokenizers/gemma_tokenizer.h src/tokenizers/gpt_oss_tokenizer.cpp src/tokenizers/gpt_oss_tokenizer.h src/tokenizers/mistral_tokenizer.cpp src/tokenizers/mistral_tokenizer.h src/tokenizers/qwen2_tokenizer.cpp src/tokenizers/qwen2_tokenizer.h src/tokenizers/t5_unigram_tokenizer.cpp src/tokenizers/t5_unigram_tokenizer.h src/tokenizers/tokenizer.cpp src/tokenizers/tokenizer.h src/tokenizers/tokenize_util.cpp src/tokenizers/tokenize_util.h src/tokenizers/vocab/vocab.h src/upscaler.cpp src/upscaler.h src/weight_manager.h SDCPP_MAIN_BASENAMES := examples/cli/image_metadata.cpp examples/cli/image_metadata.h examples/cli/main.cpp examples/cli/msf_gif.h examples/common/common.cpp examples/common/common.h examples/common/log.cpp examples/common/log.h examples/common/media_io.cpp examples/common/media_io.h examples/common/resource_owners.hpp src/tokenizers/vocab/clip_merges.hpp src/tokenizers/vocab/gemma2_merges.hpp src/tokenizers/vocab/gemma2_vocab.hpp src/tokenizers/vocab/gemma_merges.hpp src/tokenizers/vocab/gemma_vocab.hpp src/tokenizers/vocab/gpt_oss_merges.hpp src/tokenizers/vocab/gpt_oss_vocab.hpp src/tokenizers/vocab/mistral_merges.hpp src/tokenizers/vocab/mistral_vocab.hpp src/tokenizers/vocab/qwen_merges.hpp src/tokenizers/vocab/t5.hpp src/tokenizers/vocab/umt5.hpp src/tokenizers/vocab/vocab.cpp src/convert.cpp src/version.cpp diff --git a/otherarch/sdcpp/examples/cli/main.cpp b/otherarch/sdcpp/examples/cli/main.cpp index af62a9a93890..1cc7a7af4c38 100644 --- a/otherarch/sdcpp/examples/cli/main.cpp +++ b/otherarch/sdcpp/examples/cli/main.cpp @@ -754,6 +754,18 @@ int main(int argc, const char* argv[]) { return true; }; + auto load_audio = [&](const std::string& path, SDAudioOwner& audio) -> bool { + std::vector samples; + uint32_t sample_rate = 0; + uint32_t channels = 0; + if (!load_wav_from_file(path, samples, sample_rate, channels)) { + LOG_ERROR("load WAV audio from '%s' failed", path.c_str()); + return false; + } + audio.reset(std::move(samples), sample_rate, channels); + return true; + }; + if (gen_params.init_image_path.size() > 0) { if (!load_image_and_update_size(gen_params.init_image_path, gen_params.init_image)) { return 1; @@ -777,6 +789,37 @@ int main(int argc, const char* argv[]) { } } + if (!gen_params.ref_video_paths.empty()) { + gen_params.ref_videos.clear(); + gen_params.ref_videos.reserve(gen_params.ref_video_paths.size()); + for (const auto& path : gen_params.ref_video_paths) { + std::vector frames; + if (!load_images_from_dir(path, frames, 0, 0, 0, cli_params.verbose) || frames.empty()) { + LOG_ERROR("load reference video frames from '%s' failed", path.c_str()); + return 1; + } + gen_params.ref_videos.push_back(std::move(frames)); + } + + gen_params.ref_video_audios.clear(); + gen_params.ref_video_audios.resize(gen_params.ref_videos.size()); + for (size_t i = 0; i < gen_params.ref_video_audio_paths.size(); ++i) { + if (!load_audio(gen_params.ref_video_audio_paths[i], gen_params.ref_video_audios[i])) { + return 1; + } + } + } + + if (!gen_params.ref_audio_paths.empty()) { + gen_params.ref_audios.clear(); + gen_params.ref_audios.resize(gen_params.ref_audio_paths.size()); + for (size_t i = 0; i < gen_params.ref_audio_paths.size(); ++i) { + if (!load_audio(gen_params.ref_audio_paths[i], gen_params.ref_audios[i])) { + return 1; + } + } + } + if (gen_params.mask_image_path.size() > 0) { if (!load_sd_image_from_file(gen_params.mask_image.put(), gen_params.mask_image_path.c_str(), diff --git a/otherarch/sdcpp/examples/common/common.cpp b/otherarch/sdcpp/examples/common/common.cpp index 66d1d3311ae8..28f18d9e6119 100644 --- a/otherarch/sdcpp/examples/common/common.cpp +++ b/otherarch/sdcpp/examples/common/common.cpp @@ -1404,6 +1404,30 @@ ArgOptions SDGenerationParams::get_options() { return 1; }; + auto on_ref_video_arg = [&](int argc, const char** argv, int index) { + if (++index >= argc) { + return -1; + } + ref_video_paths.push_back(argv[index]); + return 1; + }; + + auto on_ref_video_audio_arg = [&](int argc, const char** argv, int index) { + if (++index >= argc) { + return -1; + } + ref_video_audio_paths.push_back(argv[index]); + return 1; + }; + + auto on_ref_audio_arg = [&](int argc, const char** argv, int index) { + if (++index >= argc) { + return -1; + } + ref_audio_paths.push_back(argv[index]); + return 1; + }; + auto on_cache_mode_arg = [&](int argc, const char** argv, int index) { if (++index >= argc) { return -1; @@ -1568,8 +1592,20 @@ ArgOptions SDGenerationParams::get_options() { on_high_noise_skip_layers_arg}, {"-r", "--ref-image", - "reference image for Flux Kontext models (can be used multiple times)", + "reference image for Flux Kontext or MiniMax-H3 Ref2VA (can be used multiple times)", on_ref_image_arg}, + {"", + "--ref-video", + "MiniMax-H3 Ref2VA reference video frame directory at 24 fps (can be used multiple times)", + on_ref_video_arg}, + {"", + "--ref-video-audio", + "WAV soundtrack paired by index with --ref-video (can be used multiple times)", + on_ref_video_audio_arg}, + {"", + "--ref-audio", + "standalone WAV reference for MiniMax-H3 Ref2VA (can be used multiple times)", + on_ref_audio_arg}, {"", "--cache-mode", "caching method: 'easycache' (DiT), 'ucache' (UNET), 'dbcache'/'taylorseer'/'cache-dit' (DiT block-level), 'spectrum' (UNET/DiT Chebyshev+Taylor forecasting)", @@ -2366,6 +2402,16 @@ bool SDGenerationParams::validate(SDMode mode) { return false; } + if (ref_video_audio_paths.size() > ref_video_paths.size()) { + LOG_ERROR("error: each --ref-video-audio needs a corresponding --ref-video"); + return false; + } + + if (mode != VID_GEN && (!ref_video_paths.empty() || !ref_video_audio_paths.empty() || !ref_audio_paths.empty())) { + LOG_ERROR("error: reference video and audio inputs require vid_gen mode"); + return false; + } + if (sample_params.shifted_timestep < 0 || sample_params.shifted_timestep > 1000) { LOG_ERROR("error: shifted_timestep must be in range [0, 1000]"); return false; @@ -2560,6 +2606,35 @@ sd_vid_gen_params_t SDGenerationParams::to_sd_vid_gen_params_t() { control_frame_views.push_back(frame.get()); } + ref_image_views.clear(); + ref_image_views.reserve(ref_images.size()); + for (auto& image : ref_images) { + ref_image_views.push_back(image.get()); + } + + ref_video_frame_views.clear(); + ref_video_frame_views.resize(ref_videos.size()); + ref_video_views.clear(); + ref_video_views.reserve(ref_videos.size()); + for (size_t i = 0; i < ref_videos.size(); ++i) { + auto& frame_views = ref_video_frame_views[i]; + frame_views.reserve(ref_videos[i].size()); + for (auto& frame : ref_videos[i]) { + frame_views.push_back(frame.get()); + } + sd_audio_t audio = i < ref_video_audios.size() ? ref_video_audios[i].get() : sd_audio_t{}; + ref_video_views.push_back({frame_views.empty() ? nullptr : frame_views.data(), + static_cast(frame_views.size()), + 24, + audio}); + } + + ref_audio_views.clear(); + ref_audio_views.reserve(ref_audios.size()); + for (auto& audio : ref_audios) { + ref_audio_views.push_back(audio.get()); + } + sample_params.guidance.slg.layers = skip_layers.empty() ? nullptr : skip_layers.data(); sample_params.guidance.slg.layer_count = skip_layers.size(); high_noise_sample_params.guidance.slg.layers = high_noise_skip_layers.empty() ? nullptr : high_noise_skip_layers.data(); @@ -2578,6 +2653,12 @@ sd_vid_gen_params_t SDGenerationParams::to_sd_vid_gen_params_t() { params.clip_skip = clip_skip; params.init_image = init_image.get(); params.end_image = end_image.get(); + params.ref_images = ref_image_views.empty() ? nullptr : ref_image_views.data(); + params.ref_images_count = static_cast(ref_image_views.size()); + params.ref_videos = ref_video_views.empty() ? nullptr : ref_video_views.data(); + params.ref_videos_count = static_cast(ref_video_views.size()); + params.ref_audios = ref_audio_views.empty() ? nullptr : ref_audio_views.data(); + params.ref_audios_count = static_cast(ref_audio_views.size()); params.control_frames = control_frame_views.empty() ? nullptr : control_frame_views.data(); params.control_frames_size = static_cast(control_frame_views.size()); params.width = get_resolved_width(); @@ -2657,6 +2738,9 @@ std::string SDGenerationParams::to_string() const { << " mask_image_path: \"" << mask_image_path << "\",\n" << " control_image_path: \"" << control_image_path << "\",\n" << " ref_image_paths: " << vec_str_to_string(ref_image_paths) << ",\n" + << " ref_video_paths: " << vec_str_to_string(ref_video_paths) << ",\n" + << " ref_video_audio_paths: " << vec_str_to_string(ref_video_audio_paths) << ",\n" + << " ref_audio_paths: " << vec_str_to_string(ref_audio_paths) << ",\n" << " control_video_path: \"" << control_video_path << "\",\n" << " auto_resize_ref_image: " << (auto_resize_ref_image ? "true" : "false") << ",\n" << " increase_ref_index: " << (increase_ref_index ? "true" : "false") << ",\n" diff --git a/otherarch/sdcpp/examples/common/common.h b/otherarch/sdcpp/examples/common/common.h index ea90c8c1bdfb..34b4a013b10c 100644 --- a/otherarch/sdcpp/examples/common/common.h +++ b/otherarch/sdcpp/examples/common/common.h @@ -212,6 +212,9 @@ struct SDGenerationParams { std::string control_image_path; std::string ip_adapter_image_path; std::vector ref_image_paths; + std::vector ref_video_paths; + std::vector ref_video_audio_paths; + std::vector ref_audio_paths; std::string control_video_path; sd_sample_params_t sample_params; @@ -275,6 +278,9 @@ struct SDGenerationParams { SDImageOwner init_image; SDImageOwner end_image; std::vector ref_images; + std::vector> ref_videos; + std::vector ref_video_audios; + std::vector ref_audios; SDImageOwner mask_image; SDImageOwner control_image; SDImageOwner ip_adapter_image; @@ -283,6 +289,9 @@ struct SDGenerationParams { // Backing storage for sd_img_gen_params_t view fields. std::vector ref_image_views; + std::vector> ref_video_frame_views; + std::vector ref_video_views; + std::vector ref_audio_views; std::vector pm_id_image_views; std::vector control_frame_views; diff --git a/otherarch/sdcpp/examples/common/media_io.cpp b/otherarch/sdcpp/examples/common/media_io.cpp index f0fdf374e444..aadec6f0f368 100644 --- a/otherarch/sdcpp/examples/common/media_io.cpp +++ b/otherarch/sdcpp/examples/common/media_io.cpp @@ -1489,3 +1489,132 @@ sd_audio_t load_pcm_wav_from_file(const std::string& path) { audio.data = samples; return audio; } + +static uint16_t read_le16(const uint8_t* data) { + return static_cast(data[0]) | + (static_cast(data[1]) << 8); +} + +static uint32_t read_le32(const uint8_t* data) { + return static_cast(data[0]) | + (static_cast(data[1]) << 8) | + (static_cast(data[2]) << 16) | + (static_cast(data[3]) << 24); +} + +bool load_wav_from_file(const std::string& path, + std::vector& interleaved_samples, + uint32_t& sample_rate, + uint32_t& channels) { + interleaved_samples.clear(); + sample_rate = 0; + channels = 0; + + std::ifstream file(path, std::ios::binary); + uint8_t riff_header[12]; + if (!file.read(reinterpret_cast(riff_header), sizeof(riff_header)) || + std::memcmp(riff_header, "RIFF", 4) != 0 || + std::memcmp(riff_header + 8, "WAVE", 4) != 0) { + return false; + } + + uint16_t audio_format = 0; + uint16_t bits_per_sample = 0; + uint16_t block_align = 0; + std::streampos data_pos = std::streampos(-1); + uint32_t data_size = 0; + + while (file.good()) { + uint8_t chunk_header[8]; + if (!file.read(reinterpret_cast(chunk_header), sizeof(chunk_header))) { + break; + } + uint32_t chunk_size = read_le32(chunk_header + 4); + std::streampos chunk_data_pos = file.tellg(); + + if (std::memcmp(chunk_header, "fmt ", 4) == 0) { + if (chunk_size < 16) { + return false; + } + std::vector fmt(chunk_size); + if (!file.read(reinterpret_cast(fmt.data()), chunk_size)) { + return false; + } + audio_format = read_le16(fmt.data()); + channels = read_le16(fmt.data() + 2); + sample_rate = read_le32(fmt.data() + 4); + block_align = read_le16(fmt.data() + 12); + bits_per_sample = read_le16(fmt.data() + 14); + if (audio_format == 0xfffe && chunk_size >= 40) { + audio_format = read_le16(fmt.data() + 24); + } + } else if (std::memcmp(chunk_header, "data", 4) == 0) { + data_pos = chunk_data_pos; + data_size = chunk_size; + file.seekg(chunk_size, std::ios::cur); + } else { + file.seekg(chunk_size, std::ios::cur); + } + + if (!file.good()) { + break; + } + if ((chunk_size & 1) != 0) { + file.seekg(1, std::ios::cur); + } + } + + const uint32_t bytes_per_sample = (bits_per_sample + 7) / 8; + if (data_pos == std::streampos(-1) || data_size == 0 || channels == 0 || sample_rate == 0 || + block_align == 0 || bytes_per_sample == 0 || block_align < channels * bytes_per_sample || + (audio_format != 1 && audio_format != 3)) { + return false; + } + + const uint64_t frame_count = data_size / block_align; + if (frame_count == 0 || frame_count > SIZE_MAX / channels) { + return false; + } + std::vector pcm(data_size); + file.clear(); + file.seekg(data_pos); + if (!file.read(reinterpret_cast(pcm.data()), data_size)) { + return false; + } + + interleaved_samples.resize(static_cast(frame_count * channels)); + for (uint64_t frame = 0; frame < frame_count; ++frame) { + const uint8_t* frame_data = pcm.data() + frame * block_align; + for (uint32_t channel = 0; channel < channels; ++channel) { + const uint8_t* sample_data = frame_data + channel * bytes_per_sample; + float sample = 0.0f; + if (audio_format == 3 && bits_per_sample == 32) { + std::memcpy(&sample, sample_data, sizeof(sample)); + } else if (audio_format == 3 && bits_per_sample == 64) { + double value; + std::memcpy(&value, sample_data, sizeof(value)); + sample = static_cast(value); + } else if (audio_format == 1 && bits_per_sample == 8) { + sample = (static_cast(sample_data[0]) - 128) / 128.0f; + } else if (audio_format == 1 && bits_per_sample == 16) { + sample = static_cast(read_le16(sample_data)) / 32768.0f; + } else if (audio_format == 1 && bits_per_sample == 24) { + int32_t value = static_cast(sample_data[0]) | + (static_cast(sample_data[1]) << 8) | + (static_cast(sample_data[2]) << 16); + if ((value & 0x800000) != 0) { + value |= ~0xffffff; + } + sample = value / 8388608.0f; + } else if (audio_format == 1 && bits_per_sample == 32) { + int32_t value = static_cast(read_le32(sample_data)); + sample = value / 2147483648.0f; + } else { + interleaved_samples.clear(); + return false; + } + interleaved_samples[static_cast(frame * channels + channel)] = sample; + } + } + return true; +} diff --git a/otherarch/sdcpp/examples/common/media_io.h b/otherarch/sdcpp/examples/common/media_io.h index df2fd019b3f8..82a6ab29a88a 100644 --- a/otherarch/sdcpp/examples/common/media_io.h +++ b/otherarch/sdcpp/examples/common/media_io.h @@ -112,4 +112,9 @@ bool write_wav_to_file(const std::string& path, sd_audio_t load_pcm_wav_from_file(const std::string& path); +bool load_wav_from_file(const std::string& path, + std::vector& interleaved_samples, + uint32_t& sample_rate, + uint32_t& channels); + #endif // __MEDIA_IO_H__ diff --git a/otherarch/sdcpp/examples/common/resource_owners.hpp b/otherarch/sdcpp/examples/common/resource_owners.hpp index d7525a5faaf8..73f8bdd81d1c 100644 --- a/otherarch/sdcpp/examples/common/resource_owners.hpp +++ b/otherarch/sdcpp/examples/common/resource_owners.hpp @@ -141,6 +141,37 @@ class SDImageOwner { } }; +class SDAudioOwner { +private: + uint32_t sample_rate_ = 0; + uint32_t channels_ = 0; + std::vector samples_; + +public: + SDAudioOwner() = default; + + void reset(std::vector samples = {}, uint32_t sample_rate = 0, uint32_t channels = 0) { + samples_ = std::move(samples); + sample_rate_ = sample_rate; + channels_ = channels; + } + + bool empty() const { + return samples_.empty(); + } + + sd_audio_t get() { + return {sample_rate_, + channels_, + channels_ == 0 ? 0 : static_cast(samples_.size() / channels_), + samples_.empty() ? nullptr : samples_.data()}; + } + + const std::vector& samples() const { + return samples_; + } +}; + class SDImageVec { private: std::vector images_; diff --git a/otherarch/sdcpp/include/stable-diffusion.h b/otherarch/sdcpp/include/stable-diffusion.h index 5af78704ee01..0a1162338fe8 100644 --- a/otherarch/sdcpp/include/stable-diffusion.h +++ b/otherarch/sdcpp/include/stable-diffusion.h @@ -247,6 +247,13 @@ typedef struct { uint8_t* data; } sd_image_t; +typedef struct { + sd_image_t* frames; + int frame_count; + int fps; + sd_audio_t audio; +} sd_ref_video_t; + typedef struct { int* layers; size_t layer_count; @@ -397,6 +404,12 @@ typedef struct { int clip_skip; sd_image_t init_image; sd_image_t end_image; + sd_image_t* ref_images; + int ref_images_count; + sd_ref_video_t* ref_videos; + int ref_videos_count; + sd_audio_t* ref_audios; + int ref_audios_count; sd_image_t* control_frames; int control_frames_size; int width; diff --git a/otherarch/sdcpp/src/conditioning/conditioner.hpp b/otherarch/sdcpp/src/conditioning/conditioner.hpp index 0b1f0d351a02..3d7ff0397f35 100644 --- a/otherarch/sdcpp/src/conditioning/conditioner.hpp +++ b/otherarch/sdcpp/src/conditioning/conditioner.hpp @@ -2,8 +2,10 @@ #define __SD_CONDITIONING_CONDITIONER_HPP__ #include +#include #include #include +#include #include "core/tensor_ggml.hpp" #include "core/util.h" @@ -25,6 +27,8 @@ struct SDCondition { sd::Tensor c_vinput_mask; std::vector>> c_image_embeds; std::vector> c_ref_images; + std::vector> c_ref_audios; + std::vector c_reference_blocks; std::vector> extra_c_crossattns; @@ -55,6 +59,12 @@ struct SDCondition { } } + for (const auto& tensor : c_ref_audios) { + if (!tensor.empty()) { + return false; + } + } + for (const auto& tensor : extra_c_crossattns) { if (!tensor.empty()) { return false; @@ -65,6 +75,18 @@ struct SDCondition { } }; +enum class MiniMaxH3PresentationKind { + IMAGE, + VIDEO, + AUDIO, +}; + +struct MiniMaxH3PresentationItem { + MiniMaxH3PresentationKind kind = MiniMaxH3PresentationKind::IMAGE; + std::vector> frames; + std::vector timestamps; +}; + static inline sd::Tensor apply_token_weights(sd::Tensor hidden_states, const std::vector& weights) { if (hidden_states.empty()) { @@ -102,11 +124,12 @@ static inline sd::Tensor apply_token_weights(sd::Tensor hidden_sta struct ConditionerParams { std::string text; - int clip_skip = -1; - int width = -1; - int height = -1; - bool zero_out_masked = false; - const std::vector>* ref_images = nullptr; // for qwen image edit + int clip_skip = -1; + int width = -1; + int height = -1; + bool zero_out_masked = false; + const std::vector>* ref_images = nullptr; // for qwen image edit + const std::vector* minimax_h3_references = nullptr; RefImageParams ref_image_params; }; @@ -1810,6 +1833,7 @@ struct LLMEmbedder : public Conditioner { sd_version_is_boogu_image(version) || sd_version_is_sefi_image(version) || sd_version_is_krea2(version) || + sd_version_is_minimax_h3(version) || sd_version_is_mage_flow(version)) { arch = LLM::LLMArch::QWEN3_VL; } else if (sd_version_is_z_image(version) || version == VERSION_OVIS_IMAGE || version == VERSION_FLUX2_KLEIN) { @@ -1992,8 +2016,10 @@ struct LLMEmbedder : public Conditioner { const std::vector>>& image_embeds, const std::set& out_layers, int prompt_template_encode_start_idx, - bool spell_quotes = false, - int max_length = 100000000) { + bool spell_quotes = false, + int max_length = 100000000, + const LLM::DeepStackImageEmbeds& deepstack_image_embeds = {}, + const std::vector& image_grids = {}) { auto tokens_weights_mask = tokenize(prompt, prompt_attn_range, min_length, max_length, spell_quotes); auto& tokens = std::get<0>(tokens_weights_mask); auto& weights = std::get<1>(tokens_weights_mask); @@ -2026,7 +2052,9 @@ struct LLMEmbedder : public Conditioner { false, false, true, - true); + true, + deepstack_image_embeds, + image_grids); GGML_ASSERT(!hidden_states.empty()); hidden_states = apply_token_weights(std::move(hidden_states), weights); GGML_ASSERT(hidden_states.shape()[1] > prompt_template_encode_start_idx); @@ -2108,6 +2136,8 @@ struct LLMEmbedder : public Conditioner { std::vector extra_prompts; std::vector> extra_prompts_attn_range; std::vector>> image_embeds; + LLM::DeepStackImageEmbeds deepstack_image_embeds; + std::vector image_grids; int prompt_template_encode_start_idx = 34; int min_length = 0; // pad tokens int max_length = 100000000; @@ -2118,7 +2148,131 @@ struct LLMEmbedder : public Conditioner { int64_t t0 = ggml_time_ms(); RefImageResizeMode resize_mode = conditioner_params.ref_image_params.vlm_resize_mode; - if (sd_version_is_hunyuan_video(version)) { + if (sd_version_is_minimax_h3(version)) { + prompt_template_encode_start_idx = 0; + out_layers = {50}; + prompt_attn_range = {0, 0}; + + if (llm->enable_vision) { + const std::string placeholder = "<|image_pad|>"; + const int patch_size = llm->config.vision.patch_size; + const int factor = patch_size * llm->config.vision.spatial_merge_size; + + auto resize_for_vision = [&](const sd::Tensor& image) { + int height = static_cast(image.shape()[1]); + int width = static_cast(image.shape()[0]); + int h_bar = std::max(factor, static_cast(std::round(static_cast(height) / factor)) * factor); + int w_bar = std::max(factor, static_cast(std::round(static_cast(width) / factor)) * factor); + resize_image_dims(height, + width, + h_bar, + w_bar, + factor, + 3136, + 12845056, + RefImageResizeMode::AREA); + auto resized = sd::ops::interpolate( + image, + std::vector{w_bar, h_bar, image.shape()[2], image.shape()[3]}); + for (int64_t i = 0; i < resized.numel(); ++i) { + resized[i] = std::clamp(resized[i], 0.f, 1.f) * 2.f - 1.f; + } + return resized; + }; + + auto add_vision_outputs = [&](std::vector> image_outputs, + int grid_h, + int grid_w) { + GGML_ASSERT(image_outputs.size() == 4); + auto image_embed = std::move(image_outputs[0]); + prompt += "<|vision_start|>"; + int image_embed_idx = static_cast(tokenizer->encode(prompt, nullptr).size()); + image_embeds.emplace_back(image_embed_idx, image_embed); + if (deepstack_image_embeds.empty()) { + deepstack_image_embeds.resize(image_outputs.size() - 1); + } + for (size_t layer = 0; layer < deepstack_image_embeds.size(); ++layer) { + deepstack_image_embeds[layer].emplace_back(image_embed_idx, std::move(image_outputs[layer + 1])); + } + image_grids.push_back({image_embed_idx, + static_cast(image_embed.shape()[1]), + grid_h, + grid_w}); + for (int64_t i = 0; i < image_embed.shape()[1]; ++i) { + prompt += placeholder; + } + prompt += "<|vision_end|>"; + }; + + const auto* references = conditioner_params.minimax_h3_references; + if (references != nullptr && !references->empty()) { + int picture_index = 0; + int video_index = 0; + int audio_index = 0; + for (const auto& item : *references) { + if (item.kind == MiniMaxH3PresentationKind::AUDIO) { + prompt += "