ver4a/llama.cpp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

CUDA: MMQ support for iq4_nl, iq4_xs (#8278)

Browse source
This commit is contained in:
Johannes Gäßler 2024-07-05 09:06:31 +02:00 • committed by GitHub
parent 0a423800ff
commit 8e558309dc
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
7 changed files with 226 additions and 80 deletions
Download patch file Download diff file Expand all files Collapse all files

8
ggml/src/ggml-cuda/mmq.cu
View file

@ -59,6 +59,12 @@ void ggml_cuda_op_mul_mat_q(
case GGML_TYPE_Q6_K:
mul_mat_q_case<GGML_TYPE_Q6_K>(ctx, args, stream);
break;
case GGML_TYPE_IQ4_XS:
mul_mat_q_case<GGML_TYPE_IQ4_XS>(ctx, args, stream);
break;
case GGML_TYPE_IQ4_NL:
mul_mat_q_case<GGML_TYPE_IQ4_NL>(ctx, args, stream);
break;
default:
GGML_ASSERT(false);
break;
@ -87,6 +93,8 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11) {
case GGML_TYPE_Q4_K:
case GGML_TYPE_Q5_K:
case GGML_TYPE_Q6_K:
case GGML_TYPE_IQ4_XS:
case GGML_TYPE_IQ4_NL:
mmq_supported = true;
break;
default: