diff --git a/api.go b/api.go index f55a602..4c98801 100644 --- a/api.go +++ b/api.go @@ -16,7 +16,7 @@ import "C" // Constant LLaMA parameters const ( - ParamContextSize = 512 // RAM requirements: 512 needs 800MB KV (~3216MB overall), 2048 needs 3200MB KV (~??? overall) + ParamContextSize = 1024 // The mem_required is 9800MB + 3216MB/state, regardless of the n_ctx size. However it does affect the KV size for persistence ParamTopK = 40 ParamTopP = 0.95 ParamTemperature = 0.08