92 lines
2.6 KiB
Nix
92 lines
2.6 KiB
Nix
{ inputs, ... }: {
|
|
|
|
flake.nixosModules.ollama = { config, lib, ... }: {
|
|
|
|
config = {
|
|
services.ollama = {
|
|
enable = true;
|
|
acceleration = "cuda";
|
|
environmentVariables = {
|
|
OLLAMA_CONTEXT_LENGTH = lib.mkDefault "16000";
|
|
};
|
|
package = let
|
|
pkgs-us = import inputs.nixpkgs-us {
|
|
system = "x86_64-linux";
|
|
config.allowUnfree = true;
|
|
};
|
|
in pkgs-us.ollama-cuda;
|
|
};
|
|
};
|
|
};
|
|
|
|
flake.nixosModules.ollama-docker = { config, lib, pkgs, ... }: let
|
|
|
|
hostPort = 11434;
|
|
|
|
subdomain = "ollama";
|
|
|
|
name = "ollama";
|
|
|
|
in {
|
|
|
|
config = {
|
|
|
|
environment.systemPackages = with pkgs; [
|
|
ollama
|
|
];
|
|
|
|
networking.firewall.interfaces = {
|
|
"ve-traefik" = {
|
|
allowedTCPPorts = [ hostPort ];
|
|
};
|
|
|
|
"ve-openwebui" = {
|
|
allowedTCPPorts = [ hostPort ];
|
|
};
|
|
};
|
|
|
|
virtualisation.oci-containers.containers.ollama = {
|
|
image = "ollama/ollama:latest";
|
|
|
|
# unstable, waiting for 26.05
|
|
#pull = "newer";
|
|
|
|
hostname = "${subdomain}.esotericbytes.com";
|
|
|
|
networks = [
|
|
"docker-main"
|
|
];
|
|
|
|
ports = [
|
|
"${builtins.toString hostPort}:11434"
|
|
];
|
|
|
|
volumes = [
|
|
"vol_ollama:/root/.ollama"
|
|
];
|
|
|
|
labels = {
|
|
"traefik.enable" = "true";
|
|
"traefik.http.routers.${name}.entrypoints" = "localsecure";
|
|
"traefik.http.routers.${name}.rule" = "Host(`${subdomain}.esotericbytes.com`)";
|
|
"traefik.http.routers.${name}.service" = "${name}";
|
|
"traefik.http.routers.${name}.tls.certResolver" = "cloudflare";
|
|
|
|
#"traefik.http.services.ollama.loadbalancer.server.url" = "http://192.168.100.10:${builtins.toString hostPort}";
|
|
"traefik.http.services.${name}.loadbalancer.server.port" = "11434";
|
|
};
|
|
|
|
extraOptions = lib.mkIf config.hardware.nvidia-container-toolkit.enable [
|
|
"--device=nvidia.com/gpu=all"
|
|
"--ip=192.168.101.22"
|
|
];
|
|
|
|
environment = {
|
|
OLLAMA_CONTEXT_LENGTH = lib.mkDefault "32000";
|
|
};
|
|
};
|
|
};
|
|
};
|
|
}
|
|
|