ver4a/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

CUDA: faster Deepseek FA, add Turing support (#13435)

Browse source
This commit is contained in:
Johannes Gäßler 2025-05-14 16:08:20 +02:00 • committed by GitHub
parent 5e7d95e22e
commit 6da34fa276
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 277 additions and 71 deletions
Download patch file Download diff file Expand all files Collapse all files

3
ggml/src/ggml-cuda/fattn.cu
View file

@ -10,6 +10,7 @@
template <int DKQ, int DV, int ncols2>
static void ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1(ggml_backend_cuda_context & ctx, ggml_tensor * dst) {
const int cc = ggml_cuda_info().devices[ggml_cuda_get_device()].cc;
const ggml_tensor * Q = dst->src[0];
if constexpr (ncols2 <= 8) {
@ -24,7 +25,7 @@ static void ggml_cuda_flash_attn_ext_mma_f16_switch_ncols1(ggml_backend_cuda_con
return;
}
if (Q->ne[1] <= 32/ncols2) {
if (ggml_cuda_highest_compiled_arch(cc) == GGML_CUDA_CC_TURING || Q->ne[1] <= 32/ncols2) {
ggml_cuda_flash_attn_ext_mma_f16_case<DKQ, DV, 32/ncols2, ncols2>(ctx, dst);
return;
}