feat: add per-module automatic flash attention selection

CMakeLists.txt

@@ -225,6 +225,8 @@ file(GLOB SD_LIB_SOURCES CONFIGURE_DEPENDS
     "src/core/*.h"
     "src/core/*.cpp"
     "src/core/*.hpp"
+    "src/core/probe/*.h"
+    "src/core/probe/*.cpp"
     "src/extensions/*.h"
     "src/extensions/*.cpp"
     "src/extensions/*.hpp"
@@ -342,6 +344,14 @@ if (SD_CUDA)
     target_link_libraries(${SD_LIB} PUBLIC CUDA::cuda_driver)
     set_property(SOURCE src/core/ggml_extend_backend.cpp APPEND PROPERTY COMPILE_DEFINITIONS SD_USE_CUDA)
 endif()
+if (SD_HIPBLAS)
+    find_package(hip REQUIRED CONFIG HINTS "$ENV{ROCM_PATH}" /opt/rocm)
+    target_link_libraries(${SD_LIB} PRIVATE hip::host)
+    set_property(SOURCE src/core/ggml_extend_backend.cpp APPEND PROPERTY COMPILE_DEFINITIONS SD_USE_HIP)
+    if (GGML_HIP_MMQ_MFMA)
+        set_property(SOURCE src/core/ggml_extend_backend.cpp APPEND PROPERTY COMPILE_DEFINITIONS SD_HIP_MFMA)
+    endif()
+endif()
 target_include_directories(${SD_LIB} PUBLIC . src include)
 target_include_directories(${SD_LIB} PRIVATE src/core)
 target_include_directories(${SD_LIB} PUBLIC . thirdparty)

docs/anima.md

@@ -15,7 +15,7 @@
 ## Examples
 
 ```sh
-.\bin\Release\sd-cli.exe --diffusion-model  ..\models\diffusion_models\anima-preview.safetensors --vae ..\models\vae\qwen_image_vae.safetensors  --llm ..\models\text_encoders\qwen_3_06b_base.safetensors  -p "a lovely cat holding a sign says 'anima.cpp'" --cfg-scale 6.0 --sampling-method euler -v --offload-to-cpu --diffusion-fa
+.\bin\Release\sd-cli.exe --diffusion-model  ..\models\diffusion_models\anima-preview.safetensors --vae ..\models\vae\qwen_image_vae.safetensors  --llm ..\models\text_encoders\qwen_3_06b_base.safetensors  -p "a lovely cat holding a sign says 'anima.cpp'" --cfg-scale 6.0 --sampling-method euler -v --offload-to-cpu
 ```
 
 <img alt="anima image example" src="../assets/anima/example.png" />

docs/animatediff.md

@@ -51,7 +51,7 @@ SD 1.5 defaults (euler_a, low CFG) produces noise-like output.
 .\bin\Release\sd-cli.exe -M vid_gen \
     --model ..\models\checkpoints\realisticVisionV60B1.safetensors \
     --motion-module ..\models\animatediff\mm_sd15_v3.safetensors \
-    --offload-to-cpu --diffusion-fa \
+    --offload-to-cpu \
     -p "a red apple on a wooden table" \
     --cfg-scale 8.0 --sampling-method euler --scheduler discrete \
     -H 512 -W 512 --video-frames 8 --fps 8 --steps 20 -s 42 \
@@ -66,7 +66,7 @@ use layer streaming:
 .\bin\Release\sd-cli.exe -M vid_gen \
     --model ..\models\checkpoints\realisticVisionV60B1.safetensors \
     --motion-module ..\models\animatediff\mm_sd15_v3.safetensors \
-    --offload-to-cpu --diffusion-fa \
+    --offload-to-cpu \
     -p "photo of coastline, rocks, storm weather, wind, waves, lightning" \
     --cfg-scale 8.0 --sampling-method euler --scheduler discrete \
     -H 384 -W 384 --video-frames 16 --fps 8 --steps 20 -s 42 \
@@ -79,7 +79,7 @@ Low-VRAM streaming (verified with a 2 GiB cap on RTX 3060):
 .\bin\Release\sd-cli.exe -M vid_gen \
     --model ..\models\checkpoints\realisticVisionV60B1.safetensors \
     --motion-module ..\models\animatediff\mm_sd15_v3.safetensors \
-    --max-vram 2.0 --diffusion-fa \
+    --max-vram 2.0 \
     -p "photo of coastline, rocks, storm weather, wind, waves, lightning" \
     --cfg-scale 8.0 --sampling-method euler --scheduler discrete \
     -H 384 -W 384 --video-frames 8 --fps 8 --steps 20 -s 42 \

docs/backend.md

@@ -215,6 +215,79 @@ If a VAE decode fails, decoding retries with spatial tiling even when `--auto-fi
 is off; supported video decoders try temporal tiling first and can then add
 spatial tiling. Spatial retries use half-size tiles along each latent dimension.
 
+## Flash attention (`--flash-attn on|off|auto`)
+
+`--flash-attn` defaults to `auto`. Selection happens while creating the model
+context, before inference, and is cached once per compute backend in that context.
+Parameter-storage backends are not benchmarked.
+
+The option also accepts module assignments, using the same module aliases as
+`--backend`:
+
+```shell
+sd-cli -m model.safetensors -p "a cat" --flash-attn auto,te=off,vae=on
+sd-cli -m model.safetensors -p "a cat" --flash-attn off,diffusion=auto
+```
+
+Supported modules are `diffusion`, `te`, `clip_vision`, `vae`, `controlnet`, and
+`audio_encoder`. Modules without an assignment use the default (`auto` if no
+default is supplied). A bare mode or `all=`, `default=`, or `*=` sets the default;
+module-specific settings take precedence regardless of their position. Within
+one assignment string, the last setting for the same module or default wins.
+Repeating `--flash-attn` replaces the entire previous string.
+
+Only modules set to `auto` need backend probing. An explicit `on` or `off` for
+one module does not change another module's decision, even when they share a
+backend. Modules without configurable attention are rejected instead of silently
+ignoring their settings.
+
+- `on` requests flash attention for all models. Unsupported attention operations
+  still fall back to ordinary attention.
+- `off` disables flash attention, including the flash-attention fallback from
+  SageAttention. SageAttention itself remains controlled by `--sage-attn`.
+- `auto` enables CPU and supported CUDA MMA configurations without timing a
+  benchmark. CUDA uses both the device's capability and the compiled GGML
+  architectures. HIP also skips timing on known CDNA MFMA architectures
+  (`gfx908`, `gfx90a`, `gfx940`/`gfx941`/`gfx942`, `gfx950`) and RDNA 3/3.5/4
+  WMMA architectures (`gfx11xx`/`gfx12xx`). The CDNA shortcut requires
+  `GGML_HIP_MMQ_MFMA` to be enabled at build time. All shortcuts still check
+  support for the FA probe graph. Vulkan devices whose description identifies
+  NVIDIA RTX hardware (GeForce RTX, TITAN RTX, Quadro RTX and professional RTX)
+  also skip timing. This is a device-name heuristic and needs no CUDA runtime.
+  Unsupported graphs are disabled; other configurations, including older or
+  unknown HIP architectures and other Vulkan devices, are benchmarked.
+
+The probe compares ordinary and flash attention through the same attention
+builder used by models, including layout conversion and F16 conversion. It uses
+two small self-attention graphs (512 tokens with head size 64, and 1024 tokens
+with head size 128, both with four heads), checks their outputs, warms up each
+path, then alternates five synchronized measurements. Flash attention is enabled
+when the sum of its median times is at least 5% lower. These representative
+shapes are a startup heuristic, not a guarantee for every model or resolution.
+Allocation/execution failures and inconclusive probes select `off`. Timing stops
+starting new measurement pairs after one second of measured computation;
+backend initialization, warmup and an in-flight pair can take longer.
+
+Different modules can select different results. A multi-device model enables
+flash attention only if all of its compute devices select `on`, preserving the
+model's existing single boolean setting. This conservative rule also covers
+row-split configurations, which can fall back to layer split during initialization.
+
+`--fa` and `--diffusion-fa` are deprecated and emit a warning when used.
+Use `--flash-attn on` and `--flash-attn off,diffusion=on`, respectively.
+Both aliases write the same mode string as `--flash-attn`; the last occurrence
+of any of these options wins. Library callers
+can set `sd_ctx_params_t::flash_attn_mode` to the same string, including
+`"on"`, `"off"`, `"auto"`, or `"auto,te=off,vae=on"`.
+A null or empty string uses automatic selection unless a
+legacy `flash_attn` or `diffusion_flash_attn` boolean is set. The context copies
+the string; no new enum is required.
+
+```shell
+sd-cli -m model.safetensors -p "a cat" --flash-attn auto
+sd-cli -m model.safetensors -p "a cat" --flash-attn off
+```
+
 ## Modules
 
 | Module | Purpose | Accepted names |

docs/backend_probe.md

@@ -0,0 +1,46 @@
+# Adding backend performance probes
+
+`src/core/probe/backend_probe.h` provides the shared measurement machinery. Operator
+policy stays separate: `src/core/probe/flash_attn.cpp`, for example, defines attention
+shapes and its CPU/CUDA/HIP/Vulkan shortcuts, then calls `sd_run_backend_probe` for unknown
+configurations. A future convolution probe can use the same runner without
+changing the attention implementation.
+
+For each representative shape, create a `BackendProbeCase`:
+
+- `build(graph, candidate)` creates inputs and either the reference or candidate
+  graph in `graph.ctx`, registers input tensors in `graph.inputs`, and assigns
+  `graph.output`. Leave the output null if the requested implementation is
+  unavailable; do not silently benchmark its fallback as the candidate.
+- By default, inputs receive deterministic F32 data, with the same seed for both
+  variants. Supply `initialize` for F16 convolution weights, integer inputs, or
+  operator-specific values. It runs after allocation, outside timed execution.
+  Both variants must receive equivalent logical inputs.
+- Default output validation requires matching shapes and contiguous F32 outputs,
+  checks finiteness, and applies absolute and relative tolerances. Supply
+  `compare` for other output types or validation rules.
+
+Graphs must be safe to execute repeatedly without modifying their inputs.
+`BackendProbeGraph` owns its GGML context and buffer, so all return paths release
+probe resources. Constructors allocate metadata only; `supported()` can also be
+used before a known-architecture shortcut, without allocating device tensors.
+
+`BackendProbeOptions` controls graph metadata capacity, warmup count, measurement
+count, measured-time budget, minimum free device memory, comparison tolerances,
+and minimum improvement. Warmup and allocation are excluded from timing. The
+runner alternates the two variants, synchronizes the backend around execution,
+and compares sums of per-case medians. The time budget is checked before each
+measurement pair; it cannot interrupt an in-flight backend operation.
+
+Only use `candidate_faster` when `completed` is true. On incomplete probes,
+`error` reports why comparison failed and the timing totals may be partial.
+The caller handles logging, known architecture rules, fallback policy, and
+context-lifetime caching. Keep caches separate by operation (or key a shared
+cache by both operation and backend); an attention result must not select a
+convolution implementation. If shapes or settings vary within one context,
+include them in the corresponding cache key as well.
+
+No convolution policy or CLI option is added by this infrastructure.
+
+Module assignment parsing remains in `src/core/ggml_extend_backend.cpp`, next
+to backend assignment parsing. The probe directory does not parse CLI/API options.

docs/boogu_image.md

@@ -17,15 +17,15 @@ Boogu Image uses a Boogu diffusion transformer, the FLUX VAE, and Qwen3-VL as th
 ### Boogu Image Base
 
 ```
-.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\boogu_image_base_bf16.safetensors --llm ..\models\text_encoders\Qwen3VL-8B-Instruct-Q4_K_M.gguf --vae ..\models\vae\ae.sft -p "a lovely cat" --diffusion-fa -v --offload-to-cpu
+.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\boogu_image_base_bf16.safetensors --llm ..\models\text_encoders\Qwen3VL-8B-Instruct-Q4_K_M.gguf --vae ..\models\vae\ae.sft -p "a lovely cat" -v --offload-to-cpu
 ```
 
 <img width="256" alt="Boogu Image Base example" src="../assets/boogu/example.png" />
 
 ### Boogu Image Edit
 
 ```
-.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\boogu_image_edit_bf16.safetensors --llm ..\models\text_encoders\Qwen3VL-8B-Instruct-Q4_K_M.gguf --llm_vision ..\models\text_encoders\mmproj-Qwen3VL-8B-Instruct-F16.gguf --vae ..\models\vae\ae.sft --diffusion-fa -v --offload-to-cpu -r ..\assets\flux\flux1-dev-q8_0.png -p "change 'flux.cpp' to 'boogu.cpp'"
+.\bin\Release\sd-cli.exe --diffusion-model ..\models\diffusion_models\boogu_image_edit_bf16.safetensors --llm ..\models\text_encoders\Qwen3VL-8B-Instruct-Q4_K_M.gguf --llm_vision ..\models\text_encoders\mmproj-Qwen3VL-8B-Instruct-F16.gguf --vae ..\models\vae\ae.sft -v --offload-to-cpu -r ..\assets\flux\flux1-dev-q8_0.png -p "change 'flux.cpp' to 'boogu.cpp'"
 ```
 
 <img width="256" alt="Boogu Image Edit example" src="../assets/boogu/edit_example.png" />

docs/ernie_image.md

@@ -21,15 +21,15 @@ You can run ERNIE-Image with stable-diffusion.cpp on GPUs with 4GB of VRAM — o
 ### ERNIE-Image-Turbo
 
 ```
-.\bin\Release\sd-cli.exe --diffusion-model  ..\models\diffusion_models\ernie-image-turbo.safetensors --vae ..\models\vae\flux2_ae.safetensors  --llm ..\models\text_encoders\ministral-3-3b.safetensors -p "a lovely cat" --cfg-scale 1.0 --steps 8 -v --offload-to-cpu --diffusion-fa
+.\bin\Release\sd-cli.exe --diffusion-model  ..\models\diffusion_models\ernie-image-turbo.safetensors --vae ..\models\vae\flux2_ae.safetensors  --llm ..\models\text_encoders\ministral-3-3b.safetensors -p "a lovely cat" --cfg-scale 1.0 --steps 8 -v --offload-to-cpu
 ```
 
 <img width="256" alt="ERNIE-Image Turbo example" src="../assets/ernie_image/turbo_example.png" />
 
 ### ERNIE-Image
 
 ```
-.\bin\Release\sd-cli.exe --diffusion-model  ..\models\diffusion_models\ernie-image-UD-Q4_K_M.gguf --vae ..\models\vae\flux2_ae.safetensors  --llm ..\models\text_encoders\ministral-3-3b.safetensors -p "a lovely cat" --cfg-scale 5.0 -v --offload-to-cpu --diffusion-fa
+.\bin\Release\sd-cli.exe --diffusion-model  ..\models\diffusion_models\ernie-image-UD-Q4_K_M.gguf --vae ..\models\vae\flux2_ae.safetensors  --llm ..\models\text_encoders\ministral-3-3b.safetensors -p "a lovely cat" --cfg-scale 5.0 -v --offload-to-cpu
 ```
 
 <img width="256" alt="ERNIE-Image example" src="../assets/ernie_image/example.png" />

docs/flux2.md

@@ -16,7 +16,7 @@
 ### Examples
 
 ```
-.\bin\Release\sd-cli.exe --diffusion-model  ..\models\diffusion_models\flux2-dev-Q4_K_S.gguf --vae ..\models\vae\flux2_ae.safetensors  --llm ..\models\text_encoders\Mistral-Small-3.2-24B-Instruct-2506-Q4_K_M.gguf -r .\kontext_input.png -p "change 'flux.cpp' to 'flux2-dev.cpp'" --cfg-scale 1.0 --sampling-method euler -v --diffusion-fa --offload-to-cpu
+.\bin\Release\sd-cli.exe --diffusion-model  ..\models\diffusion_models\flux2-dev-Q4_K_S.gguf --vae ..\models\vae\flux2_ae.safetensors  --llm ..\models\text_encoders\Mistral-Small-3.2-24B-Instruct-2506-Q4_K_M.gguf -r .\kontext_input.png -p "change 'flux.cpp' to 'flux2-dev.cpp'" --cfg-scale 1.0 --sampling-method euler -v --offload-to-cpu
 ```
 
 <img alt="flux2 example" src="../assets/flux2/example.png" />
@@ -42,19 +42,19 @@
 ### Examples
 
 ```
-.\bin\Release\sd-cli.exe --diffusion-model  ..\models\diffusion_models\flux-2-klein-4b.safetensors --vae ..\models\vae\flux2_ae.safetensors  --llm ..\models\text_encoders\qwen_3_4b.safetensors -p "a lovely cat" --cfg-scale 1.0 --steps 4 -v --offload-to-cpu --diffusion-fa
+.\bin\Release\sd-cli.exe --diffusion-model  ..\models\diffusion_models\flux-2-klein-4b.safetensors --vae ..\models\vae\flux2_ae.safetensors  --llm ..\models\text_encoders\qwen_3_4b.safetensors -p "a lovely cat" --cfg-scale 1.0 --steps 4 -v --offload-to-cpu
 ```
 
 <img alt="flux2-klein-4b" src="../assets/flux2/flux2-klein-4b.png" />
 
 ```
-.\bin\Release\sd-cli.exe --diffusion-model  ..\models\diffusion_models\flux-2-klein-4b.safetensors --vae ..\models\vae\flux2_ae.safetensors  --llm ..\models\text_encoders\qwen_3_4b.safetensors -r .\kontext_input.png -p "change 'flux.cpp' to 'klein.cpp'" --cfg-scale 1.0 --sampling-method euler -v --diffusion-fa --offload-to-cpu --steps 4
+.\bin\Release\sd-cli.exe --diffusion-model  ..\models\diffusion_models\flux-2-klein-4b.safetensors --vae ..\models\vae\flux2_ae.safetensors  --llm ..\models\text_encoders\qwen_3_4b.safetensors -r .\kontext_input.png -p "change 'flux.cpp' to 'klein.cpp'" --cfg-scale 1.0 --sampling-method euler -v --offload-to-cpu --steps 4
 ```
 
 <img alt="flux2-klein-4b-edit" src="../assets/flux2/flux2-klein-4b-edit.png" />
 
 ```
-.\bin\Release\sd-cli.exe --diffusion-model  ..\models\diffusion_models\flux-2-klein-base-4b.safetensors --vae ..\models\vae\flux2_ae.safetensors  --llm ..\models\text_encoders\qwen_3_4b.safetensors -p "a lovely cat" --cfg-scale 4.0 --steps 20 -v --offload-to-cpu --diffusion-fa
+.\bin\Release\sd-cli.exe --diffusion-model  ..\models\diffusion_models\flux-2-klein-base-4b.safetensors --vae ..\models\vae\flux2_ae.safetensors  --llm ..\models\text_encoders\qwen_3_4b.safetensors -p "a lovely cat" --cfg-scale 4.0 --steps 20 -v --offload-to-cpu
 ```
 
 <img alt="flux2-klein-base-4b" src="../assets/flux2/flux2-klein-base-4b.png" />
@@ -78,19 +78,19 @@
 ### Examples
 
 ```
-.\bin\Release\sd-cli.exe --diffusion-model  ..\models\diffusion_models\flux-2-klein-9b.safetensors --vae ..\models\vae\flux2_ae.safetensors  --llm ..\models\text_encoders\qwen_3_8b.safetensors -p "a lovely cat" --cfg-scale 1.0 --steps 4 -v --offload-to-cpu --diffusion-fa
+.\bin\Release\sd-cli.exe --diffusion-model  ..\models\diffusion_models\flux-2-klein-9b.safetensors --vae ..\models\vae\flux2_ae.safetensors  --llm ..\models\text_encoders\qwen_3_8b.safetensors -p "a lovely cat" --cfg-scale 1.0 --steps 4 -v --offload-to-cpu
 ```
 
 <img alt="flux2-klein-9b" src="../assets/flux2/flux2-klein-9b.png" />
 
 ```
-.\bin\Release\sd-cli.exe --diffusion-model  ..\models\diffusion_models\flux-2-klein-9b.safetensors --vae ..\models\vae\flux2_ae.safetensors  --llm ..\models\text_encoders\qwen_3_8b.safetensors -r .\kontext_input.png -p "change 'flux.cpp' to 'klein.cpp'" --cfg-scale 1.0 --sampling-method euler -v --diffusion-fa --offload-to-cpu --steps 4
+.\bin\Release\sd-cli.exe --diffusion-model  ..\models\diffusion_models\flux-2-klein-9b.safetensors --vae ..\models\vae\flux2_ae.safetensors  --llm ..\models\text_encoders\qwen_3_8b.safetensors -r .\kontext_input.png -p "change 'flux.cpp' to 'klein.cpp'" --cfg-scale 1.0 --sampling-method euler -v --offload-to-cpu --steps 4
 ```
 
 <img alt="flux2-klein-9b-edit" src="../assets/flux2/flux2-klein-9b-edit.png" />
 
 ```
-.\bin\Release\sd-cli.exe --diffusion-model  ..\models\diffusion_models\flux-2-klein-base-9b.safetensors --vae ..\models\vae\flux2_ae.safetensors  --llm ..\models\text_encoders\qwen_3_8b.safetensors -p "a lovely cat" --cfg-scale 4.0 --steps 20 -v --offload-to-cpu --diffusion-fa
+.\bin\Release\sd-cli.exe --diffusion-model  ..\models\diffusion_models\flux-2-klein-base-9b.safetensors --vae ..\models\vae\flux2_ae.safetensors  --llm ..\models\text_encoders\qwen_3_8b.safetensors -p "a lovely cat" --cfg-scale 4.0 --steps 20 -v --offload-to-cpu
 ```
 
 <img alt="flux2-klein-base-9b" src="../assets/flux2/flux2-klein-base-9b.png" />

docs/hunyuan_video.md

@@ -18,7 +18,7 @@ and ByT5 Small GlyphXL for glyph-aware text conditioning.
 ## Text-to-video example
 
 ```shell
-.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\models\diffusion_models\hunyuanvideo1.5_720p_t2v_fp16.safetensors --vae ..\models\vae\hunyuanvideo15_vae_fp16.safetensors --llm ..\models\text_encoders\qwen_2.5_vl_7b.safetensors --t5xxl ..\models\text_encoders\byt5_small_glyphxl_fp16.safetensors  -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -W 1280 -H 720 --offload-to-cpu --diffusion-fa --video-frames 33 --vae-tiling
+.\bin\Release\sd-cli.exe -M vid_gen --diffusion-model  ..\models\diffusion_models\hunyuanvideo1.5_720p_t2v_fp16.safetensors --vae ..\models\vae\hunyuanvideo15_vae_fp16.safetensors --llm ..\models\text_encoders\qwen_2.5_vl_7b.safetensors --t5xxl ..\models\text_encoders\byt5_small_glyphxl_fp16.safetensors  -p "a lovely cat" --cfg-scale 6.0 --sampling-method euler -v -W 1280 -H 720 --offload-to-cpu --video-frames 33 --vae-tiling
 ```
 
 <video src=../assets/hunyuan_video/hy1.5_t2v.mp4 controls="controls" muted="muted" type="video/mp4"></video>