Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 46 additions & 0 deletions cmd/minikube/cmd/start_flags.go
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,8 @@ const (
staticIP = "static-ip"
gpus = "gpus"
autoPauseInterval = "auto-pause-interval"
containerStorageRoot = "container-storage-root"
containerStorageRunRoot = "container-storage-runroot"
)

var (
Expand Down Expand Up @@ -211,6 +213,8 @@ func initMinikubeFlags() {
startCmd.Flags().String(staticIP, "", "Set a static IP for the minikube cluster, the IP must be: private, IPv4, and the last octet must be between 2 and 254, for example 192.168.200.200 (Docker and Podman drivers only)")
startCmd.Flags().StringP(gpus, "g", "", "Allow pods to use your GPUs. Options include: [all,nvidia,amd] (Docker driver with Docker container-runtime only)")
startCmd.Flags().Duration(autoPauseInterval, time.Minute*1, "Duration of inactivity before the minikube VM is paused (default 1m0s)")
startCmd.Flags().String(containerStorageRoot, "", "Absolute path for container runtime storage root directory (e.g. /host-containers/storage for CRI-O). Works with CRI-O and other compatible runtimes.")
startCmd.Flags().String(containerStorageRunRoot, "", "Absolute path for container runtime storage state directory (e.g. /host-containers/storage for CRI-O). Works with CRI-O and other compatible runtimes.")
}

// initKubernetesFlags inits the commandline flags for Kubernetes related options
Expand Down Expand Up @@ -652,13 +656,53 @@ func generateNewConfigFromFlags(cmd *cobra.Command, k8sVersion string, rtime str
MultiNodeRequested: viper.GetInt(nodes) > 1 || viper.GetBool(ha),
GPUs: viper.GetString(gpus),
AutoPauseInterval: viper.GetDuration(autoPauseInterval),
ContainerStorageRoot: viper.GetString(containerStorageRoot),
ContainerStorageRunRoot: viper.GetString(containerStorageRunRoot),
}
cc.VerifyComponents = interpretWaitFlag(*cmd)

if viper.GetString(mountString) != "" && driver.IsKIC(drvName) {
cc.ContainerVolumeMounts = []string{viper.GetString(mountString)}
}

// Validate and warn about custom storage paths
if cc.ContainerStorageRoot != "" || cc.ContainerStorageRunRoot != "" {
// Validate that custom storage paths are only used with supported runtimes
if rtime != constants.CRIO && rtime != "cri-o" {
out.WarningT("Custom container storage paths (--container-storage-root, --container-storage-runroot) are currently only supported with CRI-O runtime. The specified paths may be ignored.")
}

// Validate paths are absolute
if cc.ContainerStorageRoot != "" && !strings.HasPrefix(cc.ContainerStorageRoot, "/") {
exit.Message(reason.Usage, "--container-storage-root must be an absolute path, got: {{.path}}", out.V{"path": cc.ContainerStorageRoot})
}
if cc.ContainerStorageRunRoot != "" && !strings.HasPrefix(cc.ContainerStorageRunRoot, "/") {
exit.Message(reason.Usage, "--container-storage-runroot must be an absolute path, got: {{.path}}", out.V{"path": cc.ContainerStorageRunRoot})
}

// Automatically add storage path mounts for KIC drivers
if driver.IsKIC(drvName) {
if cc.ContainerVolumeMounts == nil {
cc.ContainerVolumeMounts = []string{}
}
if cc.ContainerStorageRoot != "" {
// Mount host storage directory to the same path in container
cc.ContainerVolumeMounts = append(cc.ContainerVolumeMounts, fmt.Sprintf("%s:%s", cc.ContainerStorageRoot, cc.ContainerStorageRoot))
}
if cc.ContainerStorageRunRoot != "" {
// Mount host runroot directory to the same path in container
cc.ContainerVolumeMounts = append(cc.ContainerVolumeMounts, fmt.Sprintf("%s:%s", cc.ContainerStorageRunRoot, cc.ContainerStorageRunRoot))
}
}

// Informational message about the feature
if rtime == constants.CRIO || rtime == "cri-o" {
if driver.IsKIC(drvName) {
out.Styled(style.Tip, "Using custom storage paths with {{.runtime}}. Storage directories will be automatically mounted.", out.V{"runtime": rtime})
}
}
}

if driver.IsKIC(drvName) {
si, err := oci.CachedDaemonInfo(drvName)
if err != nil {
Expand Down Expand Up @@ -882,6 +926,8 @@ func updateExistingConfigFromFlags(cmd *cobra.Command, existing *config.ClusterC
updateStringFromFlag(cmd, &cc.SocketVMnetClientPath, socketVMnetClientPath)
updateStringFromFlag(cmd, &cc.SocketVMnetPath, socketVMnetPath)
updateDurationFromFlag(cmd, &cc.AutoPauseInterval, autoPauseInterval)
updateStringFromFlag(cmd, &cc.ContainerStorageRoot, containerStorageRoot)
updateStringFromFlag(cmd, &cc.ContainerStorageRunRoot, containerStorageRunRoot)

if cmd.Flags().Changed(kubernetesVersion) {
kubeVer, err := getKubernetesVersion(existing)
Expand Down
2 changes: 2 additions & 0 deletions pkg/minikube/bootstrapper/kubeadm/kubeadm.go
Original file line number Diff line number Diff line change
Expand Up @@ -897,6 +897,8 @@ func (k *Bootstrapper) UpdateCluster(cfg config.ClusterConfig) error {
Runner: k.c,
Socket: cfg.KubernetesConfig.CRISocket,
KubernetesVersion: ver,
StorageRoot: cfg.ContainerStorageRoot,
StorageRunRoot: cfg.ContainerStorageRunRoot,
})
if err != nil {
return errors.Wrap(err, "runtime")
Expand Down
2 changes: 2 additions & 0 deletions pkg/minikube/config/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,8 @@ type ClusterConfig struct {
SSHAgentPID int
GPUs string
AutoPauseInterval time.Duration // Specifies interval of time to wait before checking if cluster should be paused
ContainerStorageRoot string // Custom storage root directory for container runtime (e.g., CRI-O root path)
ContainerStorageRunRoot string // Custom storage state directory for container runtime (e.g., CRI-O runroot path)
}

// KubernetesConfig contains the parameters used to configure the VM Kubernetes.
Expand Down
48 changes: 46 additions & 2 deletions pkg/minikube/cruntime/crio.go
Original file line number Diff line number Diff line change
Expand Up @@ -51,17 +51,61 @@ type CRIO struct {
ImageRepository string
KubernetesVersion semver.Version
Init sysinit.Manager
StorageRoot string // Custom storage root directory path
StorageRunRoot string // Custom storage state directory path
}

// generateCRIOConfig sets up pause image and cgroup manager for cri-o in crioConfigFile
func generateCRIOConfig(cr CommandRunner, imageRepository string, kv semver.Version, cgroupDriver string) error {
func generateCRIOConfig(cr CommandRunner, imageRepository string, kv semver.Version, cgroupDriver string, storageRoot string, storageRunRoot string) error {
pauseImage := images.Pause(kv, imageRepository)
klog.Infof("configure cri-o to use %q pause image...", pauseImage)
c := exec.Command("sh", "-c", fmt.Sprintf(`sudo sed -i 's|^.*pause_image = .*$|pause_image = %q|' %s`, pauseImage, crioConfigFile))
if _, err := cr.RunCmd(c); err != nil {
return errors.Wrap(err, "update pause_image")
}

// configure custom storage paths if provided
if storageRoot != "" || storageRunRoot != "" {
// Check if [crio] section exists, if not add it at the beginning
checkCrioSection := exec.Command("sh", "-c", fmt.Sprintf(`sudo grep -q "^\[crio\]$" %s`, crioConfigFile))
if _, err := cr.RunCmd(checkCrioSection); err != nil {
// [crio] section doesn't exist, add it at the beginning
if _, err := cr.RunCmd(exec.Command("sh", "-c", fmt.Sprintf(`sudo sed -i '1i[crio]' %s`, crioConfigFile))); err != nil {
return errors.Wrap(err, "adding [crio] section")
}
}
}

if storageRoot != "" {
// Remove any existing root configuration
if _, err := cr.RunCmd(exec.Command("sh", "-c", fmt.Sprintf(`sudo sed -i '/^root = /d' %s`, crioConfigFile))); err != nil {
return errors.Wrap(err, "removing existing root configuration")
}
// Add new root configuration after [crio] section
if _, err := cr.RunCmd(exec.Command("sh", "-c", fmt.Sprintf(`sudo sed -i '/^\[crio\]$/a root = %q' %s`, storageRoot, crioConfigFile))); err != nil {
return errors.Wrap(err, "configuring storage root")
}
}

if storageRunRoot != "" {
// Remove any existing runroot configuration
if _, err := cr.RunCmd(exec.Command("sh", "-c", fmt.Sprintf(`sudo sed -i '/^runroot = /d' %s`, crioConfigFile))); err != nil {
return errors.Wrap(err, "removing existing runroot configuration")
}
// Add new runroot configuration after [crio] section (or after root if it exists)
if storageRoot != "" {
// Add after root line
if _, err := cr.RunCmd(exec.Command("sh", "-c", fmt.Sprintf(`sudo sed -i '/^root = /a runroot = %q' %s`, storageRunRoot, crioConfigFile))); err != nil {
return errors.Wrap(err, "configuring storage runroot")
}
} else {
// Add after [crio] section
if _, err := cr.RunCmd(exec.Command("sh", "-c", fmt.Sprintf(`sudo sed -i '/^\[crio\]$/a runroot = %q' %s`, storageRunRoot, crioConfigFile))); err != nil {
return errors.Wrap(err, "configuring storage runroot")
}
}
}

// configure cgroup driver
if cgroupDriver == constants.UnknownCgroupDriver {
klog.Warningf("unable to configure cri-o to use unknown cgroup driver, will use default %q instead", constants.DefaultCgroupDriver)
Expand Down Expand Up @@ -223,7 +267,7 @@ func (r *CRIO) Enable(disOthers bool, cgroupDriver string, inUserNamespace bool)
if err := populateCRIConfig(r.Runner, r.SocketPath()); err != nil {
return err
}
if err := generateCRIOConfig(r.Runner, r.ImageRepository, r.KubernetesVersion, cgroupDriver); err != nil {
if err := generateCRIOConfig(r.Runner, r.ImageRepository, r.KubernetesVersion, cgroupDriver, r.StorageRoot, r.StorageRunRoot); err != nil {
return err
}
if err := enableIPForwarding(r.Runner); err != nil {
Expand Down
6 changes: 6 additions & 0 deletions pkg/minikube/cruntime/cruntime.go
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,10 @@ type Config struct {
InsecureRegistry []string
// GPUs add GPU devices to the container
GPUs string
// StorageRoot custom storage root directory for container runtime
StorageRoot string
// StorageRunRoot custom storage state directory for container runtime
StorageRunRoot string
}

// ListContainersOptions are the options to use for listing containers
Expand Down Expand Up @@ -238,6 +242,8 @@ func New(c Config) (Manager, error) {
ImageRepository: c.ImageRepository,
KubernetesVersion: c.KubernetesVersion,
Init: sm,
StorageRoot: c.StorageRoot,
StorageRunRoot: c.StorageRunRoot,
}, nil
case "containerd":
return &Containerd{
Expand Down
9 changes: 8 additions & 1 deletion pkg/minikube/node/node.go
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,14 @@ func teardown(cc config.ClusterConfig, name string, options *run.CommandOptions)
kv, kerr = util.ParseKubernetesVersion(cc.KubernetesConfig.KubernetesVersion)
if kerr == nil {
var crt cruntime.Manager
crt, kerr = cruntime.New(cruntime.Config{Type: cc.KubernetesConfig.ContainerRuntime, Runner: r, Socket: cc.KubernetesConfig.CRISocket, KubernetesVersion: kv})
crt, kerr = cruntime.New(cruntime.Config{
Type: cc.KubernetesConfig.ContainerRuntime,
Runner: r,
Socket: cc.KubernetesConfig.CRISocket,
KubernetesVersion: kv,
StorageRoot: cc.ContainerStorageRoot,
StorageRunRoot: cc.ContainerStorageRunRoot,
})
if kerr == nil {
sp := crt.SocketPath()
// avoid warning/error:
Expand Down
10 changes: 9 additions & 1 deletion pkg/minikube/node/start.go
Original file line number Diff line number Diff line change
Expand Up @@ -263,7 +263,13 @@ func handleNoKubernetes(starter Starter) (bool, error) {
if starter.Node.KubernetesVersion == constants.NoKubernetesVersion {
// Stop existing Kubernetes node if applicable.
if starter.StopK8s {
cr, err := cruntime.New(cruntime.Config{Type: starter.Cfg.KubernetesConfig.ContainerRuntime, Runner: starter.Runner, Socket: starter.Cfg.KubernetesConfig.CRISocket})
cr, err := cruntime.New(cruntime.Config{
Type: starter.Cfg.KubernetesConfig.ContainerRuntime,
Runner: starter.Runner,
Socket: starter.Cfg.KubernetesConfig.CRISocket,
StorageRoot: starter.Cfg.ContainerStorageRoot,
StorageRunRoot: starter.Cfg.ContainerStorageRunRoot,
})
if err != nil {
return false, err
}
Expand Down Expand Up @@ -418,6 +424,8 @@ func configureRuntimes(runner cruntime.CommandRunner, cc config.ClusterConfig, k
ImageRepository: cc.KubernetesConfig.ImageRepository,
KubernetesVersion: kv,
InsecureRegistry: cc.InsecureRegistry,
StorageRoot: cc.ContainerStorageRoot,
StorageRunRoot: cc.ContainerStorageRunRoot,
}
if cc.GPUs != "" {
co.GPUs = cc.GPUs
Expand Down