diff --git a/nixos/modules/services/web-apps/tabbyapi.nix b/nixos/modules/services/web-apps/tabbyapi.nix index a955afe463b5..7c357b15551f 100644 --- a/nixos/modules/services/web-apps/tabbyapi.nix +++ b/nixos/modules/services/web-apps/tabbyapi.nix @@ -35,12 +35,14 @@ in type = lib.types.str; default = "127.0.0.1"; description = "The IP to host on. Use 0.0.0.0 to expose on all network adapters."; + example = "0.0.0.0"; }; port = lib.mkOption { type = lib.types.port; default = 5000; description = "The port to host on."; + example = 8080; }; disable_auth = lib.mkOption { @@ -58,10 +60,34 @@ in description = "Disable fetching external content in response to requests, such as images from URLs."; }; + send_tracebacks = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + Send tracebacks over the API. + NOTE: Only enable this for debug purposes. + ''; + }; + api_servers = lib.mkOption { type = lib.types.listOf lib.types.str; default = [ "OAI" ]; description = "Select API servers to enable. Possible values: OAI, Kobold."; + example = [ + "OAI" + "Kobold" + ]; + }; + + sse_ping_interval = lib.mkOption { + type = lib.types.ints.unsigned; + default = 15; + description = '' + Seconds between SSE keep-alive pings on streaming responses. + Pings are SSE comments, ignored by compliant clients, and prevent + connections from dropping during long prefills. Set to 0 to disable. + ''; + example = 0; }; }; @@ -72,6 +98,12 @@ in description = "Enable prompt logging."; }; + log_generation_params = lib.mkOption { + type = lib.types.bool; + default = false; + description = "Enable generation parameter logging."; + }; + log_requests = lib.mkOption { type = lib.types.bool; default = false; @@ -141,6 +173,15 @@ in ''; }; + use_dummy_models = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + Sends dummy model names when the models endpoint is queried. + Enable this if the client is looking for specific OAI models. + ''; + }; + model_name = lib.mkOption { type = lib.types.nullOr lib.types.str; default = null; @@ -151,6 +192,20 @@ in example = "Qwen3_5-9B"; }; + use_as_default = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ ]; + description = '' + Names of args to use as a fallback for API load requests. + For example, if you always want cache_mode to be Q4 instead of only on the + initial model load, add "cache_mode" to this list. + ''; + example = [ + "max_seq_len" + "cache_mode" + ]; + }; + max_seq_len = lib.mkOption { type = lib.types.nullOr lib.types.int; default = null; @@ -158,6 +213,22 @@ in Max sequence length (default: min(max_position_embeddings, cache_size)). Set to -1 to fetch from the model's config.json. ''; + example = 32768; + }; + + cache_size = lib.mkOption { + type = lib.types.nullOr ( + lib.types.addCheck lib.types.ints.positive (n: lib.mod n 256 == 0) + // { + description = "positive integer, multiple of 256"; + } + ); + default = null; + description = '' + Size of the key/value cache to allocate, in tokens (default: 4096). + Must be a multiple of 256. + ''; + example = 32768; }; cache_mode = lib.mkOption { @@ -168,6 +239,7 @@ in Specify the pair k_bits,v_bits where k_bits and v_bits are integers from 2-8 (e.g. '8,8'). The legacy values 'FP16', 'Q8', 'Q6', 'Q4' are also accepted. ''; + example = "8,8"; }; tensor_parallel = lib.mkOption { @@ -185,6 +257,111 @@ in description = "Automatically allocate resources to GPUs. Not parsed for single GPU users."; }; + autosplit_reserve = lib.mkOption { + type = lib.types.listOf lib.types.number; + default = [ 96 ]; + description = '' + Reserve VRAM used for autosplit loading, as a list of MB per GPU + (default: 96 MB on GPU 0). + ''; + example = [ + 96 + 96 + ]; + }; + + gpu_split = lib.mkOption { + type = lib.types.listOf lib.types.number; + default = [ ]; + description = '' + List of VRAM sizes to split between GPUs, in GB. + Used with tensor parallelism. + ''; + example = [ + 16 + 24 + ]; + }; + + cpu_moe_offload_layers = lib.mkOption { + type = lib.types.ints.unsigned; + default = 0; + description = '' + Number of mixture-of-expert layers to offload to CPU inference. + Only affects MoE models. Set a large value such as 999 to offload all layers. + ''; + example = 999; + }; + + rope_scale = lib.mkOption { + type = lib.types.nullOr lib.types.number; + default = 1.0; + description = '' + Rope scale, same as compress_pos_emb. + Use if the model was trained on long context with rope. + Set to null to pull the value from the model. + + NOTE: If a model has YaRN rope scaling, it will automatically be enabled by + ExLlama and the rope_scale and rope_alpha settings won't apply. + ''; + example = 4.0; + }; + + rope_alpha = lib.mkOption { + type = lib.types.nullOr (lib.types.either lib.types.number (lib.types.enum [ "auto" ])); + default = null; + description = '' + Rope alpha, same as alpha_value. Set to "auto" to auto-calculate. + Leaving this null will either pull from the model or auto-calculate. + ''; + example = "auto"; + }; + + chunk_size = lib.mkOption { + type = lib.types.ints.positive; + default = 2048; + description = '' + Chunk size for prompt ingestion. + A lower value reduces VRAM usage but decreases ingestion speed. + NOTE: Effects vary depending on the model. An ideal value is between 512 and 4096. + ''; + example = 512; + }; + + output_chunking = lib.mkOption { + type = lib.types.bool; + default = true; + description = '' + Use output chunking. Instead of allocating cache space for the entire completion + at once, allocate in chunks as needed. Used by EXL3 models only. + ''; + }; + + max_batch_size = lib.mkOption { + type = lib.types.nullOr lib.types.ints.positive; + default = null; + description = '' + Set the maximum number of generation jobs that can run concurrently. + The default maximum batch size for transformer architectures is 32. Recurrent + models with linear or sliding attention use more VRAM to support larger batches, + so the default value is reduced to 4. If you do not require concurrency at all, + you can reduce it further to minimize VRAM overhead. + ''; + example = 1; + }; + + prompt_template = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + description = '' + Set the prompt template for this model. + If null, attempts to look for the model's chat template. + If a model contains multiple templates in its tokenizer_config.json, + set this to the name of the template you want to use. + NOTE: Only works with chat completion message lists! + ''; + }; + dummy_model_names = lib.mkOption { type = lib.types.listOf lib.types.str; default = [ "gpt-3.5-turbo" ]; @@ -192,6 +369,10 @@ in A list of fake model names that are sent via the /v1/models endpoint. Also used as bypasses for strict mode if inline_model_loading is true. ''; + example = [ + "gpt-3.5-turbo" + "gpt-4" + ]; }; vision = lib.mkOption { @@ -200,6 +381,34 @@ in description = "Enables vision support if the model supports it."; }; + template_vars_default = lib.mkOption { + type = lib.types.attrsOf lib.types.anything; + default = { }; + description = '' + Default chat template variables. Merged into the template variables of every + chat completion request; values sent by the client (template_vars / + chat_template_kwargs, or the top-level reasoning_effort field) take precedence. + Use for model-specific reasoning knobs. + ''; + example = { + enable_thinking = true; + }; + }; + + template_vars_force = lib.mkOption { + type = lib.types.attrsOf lib.types.anything; + default = { }; + description = '' + Forced chat template variables. Like template_vars_default, but these override + any values sent by the client. + Replaces the deprecated force_enable_thinking option, which is still accepted + as an alias for { enable_thinking = true; }. + ''; + example = { + reasoning_effort = "high"; + }; + }; + reasoning = lib.mkOption { type = lib.types.bool; default = false; @@ -220,7 +429,243 @@ in default = ""; description = "The end token for reasoning content."; }; + + start_in_reasoning = lib.mkOption { + type = lib.types.enum [ + "auto" + "always" + "never" + ]; + default = "auto"; + description = '' + Whether generation starts inside a reasoning block. + "auto" guesses by scanning the end of the templated prompt for an unclosed + reasoning start token. + ''; + example = "always"; + }; + + tool_calls_in_reasoning = lib.mkOption { + type = lib.types.bool; + default = true; + description = '' + Parse tool calls that occur inside reasoning content. + If false, tool call tags inside a reasoning block are treated as plain + reasoning text. + ''; + }; + + tool_format = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + description = '' + Tool format, e.g. "qwen3_coder". See upstream docs for supported formats. + If null, tool calls from the model will not be parsed by the server. + ''; + example = "qwen3_coder"; + }; + + harmony = lib.mkOption { + type = lib.types.nullOr lib.types.bool; + default = null; + description = '' + Parse responses in the Harmony message format (gpt-oss models). + Auto-detected from the model's special tokens when null; set to true or false + to override. Setting tool_format to "harmony" is equivalent to setting this to + true. When active, supersedes the reasoning and tool format settings. + ''; + }; }; + + draft_model = { + draft_mode = lib.mkOption { + type = lib.types.enum [ + "model" + "disabled" + "mtp" + "ngram" + ]; + default = "model"; + description = '' + Drafting mode for exllamav3. + In "model" mode, drafting is disabled if no draft_model_name is provided. + ''; + example = "ngram"; + }; + + draft_model_dir = lib.mkOption { + type = lib.types.str; + default = "models"; + description = "Directory to look for draft models. Relative to the state directory."; + example = "drafts"; + }; + + draft_model_name = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + description = '' + An initial draft model to load. + Ensure the model is in the draft model directory. + ''; + example = "Qwen3-0.6B-exl3"; + }; + + draft_rope_scale = lib.mkOption { + type = lib.types.nullOr lib.types.number; + default = 1.0; + description = '' + Rope scale for draft models, same as compress_pos_emb. + Use if the draft model was trained on long context with rope. + ''; + example = 4.0; + }; + + draft_rope_alpha = lib.mkOption { + type = lib.types.nullOr lib.types.number; + default = null; + description = '' + Rope alpha for draft models, same as alpha_value. + Leaving this null will either pull from the model or auto-calculate. + ''; + example = 2.0; + }; + + draft_cache_mode = lib.mkOption { + type = lib.types.enum [ + "FP16" + "Q8" + "Q6" + "Q4" + ]; + default = "FP16"; + description = '' + Cache mode for draft models to save VRAM. + Unlike the model's cache_mode, this does not accept a k_bits,v_bits pair. + ''; + example = "Q8"; + }; + + draft_gpu_split = lib.mkOption { + type = lib.types.listOf lib.types.number; + default = [ ]; + description = '' + List of VRAM sizes to split between GPUs, in GB. + If this is empty, the draft model is autosplit. + ''; + example = [ + 2 + 2 + ]; + }; + + draft_num_tokens = lib.mkOption { + type = lib.types.nullOr lib.types.ints.positive; + default = null; + description = '' + Number of tokens to draft per iteration (default: draft model default). + Recurrent (linear or sliding attention) models use more VRAM for longer drafts. + This overhead multiplies with the max batch size, so for models with long drafts + (e.g. DFlash with 15 tokens by default) shorter drafts may be preferable. + ''; + example = 4; + }; + + dynamic_draft = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + Adjust number of draft tokens dynamically based on observed acceptance rates. + Ceiling is given by draft_num_tokens. + ''; + }; + + ngram_match_min = lib.mkOption { + type = lib.types.ints.positive; + default = 2; + description = '' + Minimum match length for exllamav3 n-gram drafting. + Only used when draft_mode is "ngram". + ''; + example = 3; + }; + }; + + sampling = { + override_preset = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + description = '' + Select a sampler override preset, found in the sampler-overrides folder. + This overrides default fallbacks for sampler values that are passed to the API. + NOTE: "safe_defaults" is noob friendly and provides fallbacks for frontends that + don't send sampling parameters. Leave this null for any advanced usage. + ''; + example = "safe_defaults"; + }; + }; + + lora = { + lora_dir = lib.mkOption { + type = lib.types.str; + default = "loras"; + description = "Directory to look for LoRAs. Relative to the state directory."; + }; + + loras = lib.mkOption { + type = lib.types.listOf ( + lib.types.submodule { + options = { + name = lib.mkOption { + type = lib.types.str; + description = "Name of the LoRA directory inside lora_dir."; + }; + + scaling = lib.mkOption { + type = lib.types.number; + default = 1.0; + description = "Scaling factor for this LoRA."; + }; + }; + } + ); + default = [ ]; + description = "List of LoRAs to load and associated scaling factors."; + example = [ + { + name = "lora1"; + scaling = 1.0; + } + ]; + }; + }; + + memory = { + sysmem_recurrent_cache = lib.mkOption { + type = lib.types.ints.unsigned; + default = 4096; + description = "Max size of recurrent cache in system memory, in MB."; + example = 8192; + }; + + sysmem_kv_cache = lib.mkOption { + type = lib.types.ints.unsigned; + default = 0; + description = "Size of system memory second-tier key/value cache, in MB."; + example = 4096; + }; + + cuda_malloc_async = lib.mkOption { + type = lib.types.bool; + default = true; + description = '' + Use the cudaMallocAsync backend in Torch. + Enabling this is generally preferable, but it may cause issues with certain + workloads. Try disabling it if you experience intermittent OoM errors. If false, + Torch will use the allocator defined by the system environment. + ''; + }; + }; + }; }; }; @@ -236,6 +681,14 @@ in services.tabbyapi.package = pkgs.pkgsCuda.tabbyapi; ''; } + { + assertion = !(cfg.settings.model ? force_enable_thinking); + message = '' + services.tabbyapi.settings.model.force_enable_thinking is deprecated upstream. + Use template_vars_force instead: + services.tabbyapi.settings.model.template_vars_force.enable_thinking = true; + ''; + } ]; networking.firewall.allowedTCPPorts = lib.mkIf cfg.openFirewall [ cfg.settings.network.port