gian/incus-mirror
1
0
Fork 0
mirror of https://github.com/lxc/incus synced 2026-08-02 05:26:46 +00:00
Code Issues Actions 4 Packages Projects Releases Wiki Activity
c6f01c862f
incus-mirror/internal/server/instance/drivers/driver_lxc.go
Stéphane Graber 23066254f5
incusd/instance: Fix NVIDIA require.cuda and require.driver handling 
The condition was inverted, so a configured value was silently dropped
while an empty value wrote an empty environment variable. Only write the
variable when a value is set.

Signed-off-by: Stéphane Graber <stgraber@stgraber.org>
2026-07-30 09:45:44 -04:00

9848 lines
265 KiB
Go
Raw Blame History

package drivers
import (
"bufio"
"bytes"
"context"
"database/sql"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io"
"io/fs"
"maps"
"net"
"net/http"
"os"
"os/exec"
"path"
"path/filepath"
"runtime"
"slices"
"sort"
"strconv"
"strings"
"sync"
"syscall"
"time"
"github.com/checkpoint-restore/go-criu/v8/crit"
"github.com/flosch/pongo2/v6"
"github.com/google/uuid"
"github.com/gorilla/websocket"
"github.com/kballard/go-shellquote"
liblxc "github.com/lxc/go-lxc"
ociSpecs "github.com/opencontainers/runtime-spec/specs-go"
"github.com/pkg/sftp"
yaml "go.yaml.in/yaml/v4"
"golang.org/x/sync/errgroup"
"golang.org/x/sys/unix"
"google.golang.org/protobuf/proto"
internalInstance "github.com/lxc/incus/v7/internal/instance"
"github.com/lxc/incus/v7/internal/instancewriter"
internalIO "github.com/lxc/incus/v7/internal/io"
"github.com/lxc/incus/v7/internal/jmap"
"github.com/lxc/incus/v7/internal/linux"
"github.com/lxc/incus/v7/internal/migration"
"github.com/lxc/incus/v7/internal/netutils"
"github.com/lxc/incus/v7/internal/rsync"
"github.com/lxc/incus/v7/internal/server/apparmor"
"github.com/lxc/incus/v7/internal/server/cgroup"
"github.com/lxc/incus/v7/internal/server/daemon"
"github.com/lxc/incus/v7/internal/server/db"
"github.com/lxc/incus/v7/internal/server/db/cluster"
"github.com/lxc/incus/v7/internal/server/db/operationtype"
"github.com/lxc/incus/v7/internal/server/device"
deviceConfig "github.com/lxc/incus/v7/internal/server/device/config"
"github.com/lxc/incus/v7/internal/server/device/nictype"
"github.com/lxc/incus/v7/internal/server/instance"
"github.com/lxc/incus/v7/internal/server/instance/instancetype"
"github.com/lxc/incus/v7/internal/server/instance/operationlock"
"github.com/lxc/incus/v7/internal/server/ip"
"github.com/lxc/incus/v7/internal/server/lifecycle"
"github.com/lxc/incus/v7/internal/server/locking"
"github.com/lxc/incus/v7/internal/server/metrics"
localMigration "github.com/lxc/incus/v7/internal/server/migration"
"github.com/lxc/incus/v7/internal/server/network"
"github.com/lxc/incus/v7/internal/server/operations"
"github.com/lxc/incus/v7/internal/server/project"
"github.com/lxc/incus/v7/internal/server/response"
"github.com/lxc/incus/v7/internal/server/seccomp"
"github.com/lxc/incus/v7/internal/server/selinux"
"github.com/lxc/incus/v7/internal/server/state"
storagePools "github.com/lxc/incus/v7/internal/server/storage"
storageDrivers "github.com/lxc/incus/v7/internal/server/storage/drivers"
localUtil "github.com/lxc/incus/v7/internal/server/util"
internalUtil "github.com/lxc/incus/v7/internal/util"
"github.com/lxc/incus/v7/shared/api"
"github.com/lxc/incus/v7/shared/idmap"
"github.com/lxc/incus/v7/shared/ioprogress"
"github.com/lxc/incus/v7/shared/logger"
"github.com/lxc/incus/v7/shared/osarch"
"github.com/lxc/incus/v7/shared/osinfo"
"github.com/lxc/incus/v7/shared/revert"
"github.com/lxc/incus/v7/shared/subprocess"
"github.com/lxc/incus/v7/shared/termios"
"github.com/lxc/incus/v7/shared/units"
"github.com/lxc/incus/v7/shared/util"
"github.com/lxc/incus/v7/shared/ws"
)
// OCINetworkInterface is the expected network configuration for an OCI container interface.
// It is consumed by the forknet dhcp process through the container's interfaces.json file.
type OCINetworkInterface struct {
DHCP4 bool `json:"dhcp4"`
DHCP6 bool `json:"dhcp6"`
Route4 bool `json:"route4"`
Route6 bool `json:"route6"`
}
// Helper functions.
func lxcSetConfigItem(c *liblxc.Container, key string, value string) error {
if c == nil {
return errors.New("Uninitialized go-lxc struct")
}
if !liblxc.RuntimeLiblxcVersionAtLeast(liblxc.Version(), 2, 1, 0) {
switch key {
case "lxc.uts.name":
key = "lxc.utsname"
case "lxc.pty.max":
key = "lxc.pts"
case "lxc.tty.dir":
key = "lxc.devttydir"
case "lxc.tty.max":
key = "lxc.tty"
case "lxc.apparmor.profile":
key = "lxc.aa_profile"
case "lxc.apparmor.allow_incomplete":
key = "lxc.aa_allow_incomplete"
case "lxc.selinux.context":
key = "lxc.se_context"
case "lxc.mount.fstab":
key = "lxc.mount"
case "lxc.console.path":
key = "lxc.console"
case "lxc.seccomp.profile":
key = "lxc.seccomp"
case "lxc.signal.halt":
key = "lxc.haltsignal"
case "lxc.signal.reboot":
key = "lxc.rebootsignal"
case "lxc.signal.stop":
key = "lxc.stopsignal"
case "lxc.log.syslog":
key = "lxc.syslog"
case "lxc.log.level":
key = "lxc.loglevel"
case "lxc.log.file":
key = "lxc.logfile"
case "lxc.init.cmd":
key = "lxc.init_cmd"
case "lxc.init.uid":
key = "lxc.init_uid"
case "lxc.init.gid":
key = "lxc.init_gid"
case "lxc.idmap":
key = "lxc.id_map"
}
}
if strings.HasPrefix(key, "lxc.prlimit.") {
if !liblxc.RuntimeLiblxcVersionAtLeast(liblxc.Version(), 2, 1, 0) {
return errors.New(`Process limits require liblxc >= 2.1`)
}
}
err := c.SetConfigItem(key, value)
if err != nil {
return fmt.Errorf("Failed to set LXC config: %s=%s", key, value)
}
return nil
}
// lxcEncodeCmd encodes a command for lxc.init.cmd/lxc.execute.cmd, whose
// parser only supports whole-word quoting with no escape sequences.
func lxcEncodeCmd(args []string) (string, error) {
words := make([]string, 0, len(args))
for _, arg := range args {
if arg != "" && !strings.ContainsAny(arg, " \t\n\v\f\r") && !strings.HasPrefix(arg, "'") && !strings.HasPrefix(arg, "\"") {
words = append(words, arg)
continue
}
if !strings.Contains(arg, "\"") {
words = append(words, "\""+arg+"\"")
continue
}
if !strings.Contains(arg, "'") {
words = append(words, "'"+arg+"'")
continue
}
return "", fmt.Errorf("Unable to encode command argument: %q", arg)
}
return strings.Join(words, " "), nil
}
func lxcStatusCode(lxcState liblxc.State) api.StatusCode {
return map[int]api.StatusCode{
1: api.Stopped,
2: api.Starting,
3: api.Running,
4: api.Stopping,
5: api.Aborting,
6: api.Freezing,
7: api.Frozen,
8: api.Thawed,
9: api.Error,
}[int(lxcState)]
}
// lxcCreate creates the DB storage records and sets up instance devices.
// Returns a revert fail function that can be used to undo this function if a subsequent step fails.
func lxcCreate(s *state.State, args db.InstanceArgs, p api.Project, partialDeviceValidation bool, op *operations.Operation) (instance.Instance, revert.Hook, error) {
reverter := revert.New()
defer reverter.Fail()
// Create the container struct
d := &lxc{
common: common{
state: s,
op: op,
architecture: args.Architecture,
creationDate: args.CreationDate,
dbType: args.Type,
description: args.Description,
ephemeral: args.Ephemeral,
expiryDate: args.ExpiryDate,
id: args.ID,
lastUsedDate: args.LastUsedDate,
localConfig: args.Config,
localDevices: args.Devices,
logger: logger.AddContext(logger.Ctx{"instanceType": args.Type, "instance": args.Name, "project": args.Project}),
name: args.Name,
node: args.Node,
profiles: args.Profiles,
project: p,
isSnapshot: args.Snapshot,
stateful: args.Stateful,
},
}
// Cleanup the zero values
if d.expiryDate.IsZero() {
d.expiryDate = time.Time{}
}
if d.creationDate.IsZero() {
d.creationDate = time.Time{}
}
if d.lastUsedDate.IsZero() {
d.lastUsedDate = time.Time{}
}
if args.Snapshot {
d.logger.Info("Creating instance snapshot", logger.Ctx{"ephemeral": d.ephemeral})
} else {
d.logger.Info("Creating instance", logger.Ctx{"ephemeral": d.ephemeral})
}
// Load the config.
err := d.init()
if err != nil {
return nil, nil, fmt.Errorf("Failed to expand config: %w", err)
}
// When not a snapshot, perform full validation.
if !args.Snapshot {
// Validate expanded config (allows mixed instance types for profiles).
err = instance.ValidConfig(s.OS, d.expandedConfig, true, instancetype.Any)
if err != nil {
return nil, nil, fmt.Errorf("Invalid config: %w", err)
}
err = instance.ValidDevices(s, d.project, d.Type(), d.localDevices, d.expandedDevices)
if err != nil {
return nil, nil, fmt.Errorf("Invalid devices: %w", err)
}
}
_, rootDiskDevice, err := d.getRootDiskDevice()
if err != nil {
return nil, nil, fmt.Errorf("Failed getting root disk: %w", err)
}
if rootDiskDevice["pool"] == "" {
return nil, nil, errors.New("The instance's root device is missing the pool property")
}
// Initialize the storage pool.
d.storagePool, err = storagePools.LoadByName(d.state, rootDiskDevice["pool"])
if err != nil {
return nil, nil, fmt.Errorf("Failed loading storage pool: %w", err)
}
volType, err := storagePools.InstanceTypeToVolumeType(d.Type())
if err != nil {
return nil, nil, err
}
storagePoolSupported := slices.Contains(d.storagePool.Driver().Info().VolumeTypes, volType)
if !storagePoolSupported {
return nil, nil, errors.New("Storage pool does not support instance type")
}
// Setup the initial idmap config.
var idmapSet *idmap.Set
base := int64(0)
if !d.IsPrivileged() {
idmapSet, base, err = d.findIdmap()
if err != nil {
return nil, nil, err
}
}
idmapSetJSON, err := idmapSet.ToJSON()
if err != nil {
return nil, nil, fmt.Errorf("Failed to encode ID map: %w", err)
}
v := map[string]string{
"volatile.idmap.next": idmapSetJSON,
"volatile.idmap.base": fmt.Sprintf("%v", base),
}
// Invalidate the idmap cache.
d.idmapset = nil
// Set last_state if not currently set.
if d.localConfig["volatile.last_state.idmap"] == "" {
v["volatile.last_state.idmap"] = "[]"
}
err = d.VolatileSet(v)
if err != nil {
return nil, nil, err
}
// Re-run init to update the idmap.
err = d.init()
if err != nil {
return nil, nil, err
}
if !d.IsSnapshot() {
// Add devices to container.
cleanup, err := d.devicesAdd(d, false, partialDeviceValidation)
if err != nil {
return nil, nil, err
}
reverter.Add(cleanup)
}
if d.isSnapshot {
d.logger.Info("Created instance snapshot", logger.Ctx{"ephemeral": d.ephemeral})
} else {
d.logger.Info("Created instance", logger.Ctx{"ephemeral": d.ephemeral})
}
if d.isSnapshot {
d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceSnapshotCreated.Event(d, nil))
} else {
// Add instance to authorizer.
err = d.state.Authorizer.AddInstance(d.state.ShutdownCtx, d.project.Name, d.Name())
if err != nil {
logger.Error("Failed to add instance to authorizer", logger.Ctx{"instanceName": d.Name(), "projectName": d.project.Name, "error": err})
}
reverter.Add(func() { _ = d.state.Authorizer.DeleteInstance(d.state.ShutdownCtx, d.project.Name, d.Name()) })
d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceCreated.Event(d, map[string]any{
"type": api.InstanceTypeContainer,
"storage-pool": d.storagePool.Name(),
"location": d.Location(),
}))
}
cleanup := reverter.Clone().Fail
reverter.Success()
return d, cleanup, err
}
func lxcLoad(s *state.State, args db.InstanceArgs, p api.Project) (instance.Instance, error) {
// Create the container struct
d := lxcInstantiate(s, args, nil, p)
// Expand config and devices
err := d.(*lxc).expandConfig()
if err != nil {
return nil, err
}
return d, nil
}
// Unload is called by the garbage collector.
func lxcUnload(d *lxc) {
d.release()
}
// release releases any internal reference to a liblxc container, invalidating the go-lxc cache.
func (d *lxc) release() {
d.cMu.Lock()
defer d.cMu.Unlock()
if d.c != nil {
_ = d.c.Release()
d.c = nil
}
}
// Create a container struct without initializing it.
func lxcInstantiate(s *state.State, args db.InstanceArgs, expandedDevices deviceConfig.Devices, p api.Project) instance.Instance {
d := &lxc{
common: common{
state: s,
architecture: args.Architecture,
creationDate: args.CreationDate,
dbType: args.Type,
description: args.Description,
ephemeral: args.Ephemeral,
expiryDate: args.ExpiryDate,
id: args.ID,
lastUsedDate: args.LastUsedDate,
localConfig: args.Config,
localDevices: args.Devices,
logger: logger.AddContext(logger.Ctx{"instanceType": args.Type, "instance": args.Name, "project": args.Project}),
name: args.Name,
node: args.Node,
profiles: args.Profiles,
project: p,
isSnapshot: args.Snapshot,
stateful: args.Stateful,
},
}
// Cleanup the zero values
if d.expiryDate.IsZero() {
d.expiryDate = time.Time{}
}
if d.creationDate.IsZero() {
d.creationDate = time.Time{}
}
if d.lastUsedDate.IsZero() {
d.lastUsedDate = time.Time{}
}
// This is passed during expanded config validation.
if expandedDevices != nil {
d.expandedDevices = expandedDevices
}
return d
}
// The LXC container driver.
type lxc struct {
common
// Config handling.
fromHook bool
cMu sync.Mutex
cFinalizer sync.Once
// Cached handles.
// Do not use these variables directly, instead use their associated get functions so they
// will be initialized on demand.
c *liblxc.Container // Use d.initLXC() instead of accessing this directly.
cConfig bool
idmapset *idmap.Set
}
var idmapLock sync.Mutex
func (d *lxc) findIdmap() (*idmap.Set, int64, error) {
if d.state.OS.IdmapSet == nil {
return nil, 0, errors.New("System doesn't have a functional idmap setup")
}
idmapSize := func(size string) (int64, error) {
var idMapSize int64
if size == "" || size == "auto" {
if util.IsTrue(d.expandedConfig["security.idmap.isolated"]) {
idMapSize = 65536
} else {
if len(d.state.OS.IdmapSet.Entries) != 2 {
return 0, fmt.Errorf("Bad initial idmap: %v", d.state.OS.IdmapSet)
}
idMapSize = d.state.OS.IdmapSet.Entries[0].MapRange
}
} else {
size, err := strconv.ParseInt(size, 10, 64)
if err != nil {
return 0, err
}
idMapSize = size
}
return idMapSize, nil
}
rawMaps, err := idmap.NewSetFromIncusIDMap(d.expandedConfig["raw.idmap"])
if err != nil {
return nil, 0, err
}
mkIdmap := func(offset int64, size int64) (*idmap.Set, error) {
set := &idmap.Set{Entries: []idmap.Entry{
{IsUID: true, NSID: 0, HostID: offset, MapRange: size},
{IsGID: true, NSID: 0, HostID: offset, MapRange: size},
}}
for _, ent := range rawMaps.Entries {
err := set.AddSafe(ent)
if err != nil && errors.Is(err, idmap.ErrHostIDIsSubID) {
return nil, err
}
}
return set, nil
}
if !util.IsTrue(d.expandedConfig["security.idmap.isolated"]) {
// Create a new set based from the global one.
newIdmapset := idmap.Set{Entries: make([]idmap.Entry, len(d.state.OS.IdmapSet.Entries))}
copy(newIdmapset.Entries, d.state.OS.IdmapSet.Entries)
// Restrict the range sizes if specified.
if d.expandedConfig["security.idmap.size"] != "" {
size, err := idmapSize(d.expandedConfig["security.idmap.size"])
if err != nil {
return nil, 0, err
}
for k, ent := range newIdmapset.Entries {
if ent.MapRange < size {
continue
}
newIdmapset.Entries[k].MapRange = size
}
}
// Apply the raw idmap entries.
for _, ent := range rawMaps.Entries {
err := newIdmapset.AddSafe(ent)
if err != nil && errors.Is(err, idmap.ErrHostIDIsSubID) {
return nil, 0, err
}
}
return &newIdmapset, 0, nil
}
size, err := idmapSize(d.expandedConfig["security.idmap.size"])
if err != nil {
return nil, 0, err
}
if d.expandedConfig["security.idmap.base"] != "" {
offset, err := strconv.ParseInt(d.expandedConfig["security.idmap.base"], 10, 64)
if err != nil {
return nil, 0, err
}
set, err := mkIdmap(offset, size)
if err != nil && errors.Is(err, idmap.ErrHostIDIsSubID) {
return nil, 0, err
}
return set, offset, nil
}
idmapLock.Lock()
defer idmapLock.Unlock()
cts, err := instance.LoadNodeAll(d.state, instancetype.Container)
if err != nil {
return nil, 0, err
}
offset := d.state.OS.IdmapSet.Entries[0].HostID + 65536
mapentries := idmap.ByHostID{}
for _, container := range cts {
if container.Type() != instancetype.Container {
continue
}
/* Don't change our map Just Because. */
if container.ID() == d.id {
continue
}
if container.IsPrivileged() {
continue
}
if util.IsFalseOrEmpty(container.ExpandedConfig()["security.idmap.isolated"]) {
continue
}
if container.ExpandedConfig()["volatile.idmap.base"] == "" {
continue
}
cBase, err := strconv.ParseInt(container.ExpandedConfig()["volatile.idmap.base"], 10, 64)
if err != nil {
return nil, 0, err
}
cSize, err := idmapSize(container.ExpandedConfig()["security.idmap.size"])
if err != nil {
return nil, 0, err
}
mapentries.Entries = append(mapentries.Entries, idmap.Entry{HostID: int64(cBase), MapRange: cSize})
}
sort.Sort(mapentries)
for i := range mapentries.Entries {
if i == 0 {
if mapentries.Entries[0].HostID < offset+size {
offset = mapentries.Entries[0].HostID + mapentries.Entries[0].MapRange
continue
}
set, err := mkIdmap(offset, size)
if err != nil && errors.Is(err, idmap.ErrHostIDIsSubID) {
return nil, 0, err
}
return set, offset, nil
}
if mapentries.Entries[i-1].HostID+mapentries.Entries[i-1].MapRange > offset {
offset = mapentries.Entries[i-1].HostID + mapentries.Entries[i-1].MapRange
continue
}
offset = mapentries.Entries[i-1].HostID + mapentries.Entries[i-1].MapRange
if offset+size < mapentries.Entries[i].HostID {
set, err := mkIdmap(offset, size)
if err != nil && errors.Is(err, idmap.ErrHostIDIsSubID) {
return nil, 0, err
}
return set, offset, nil
}
offset = mapentries.Entries[i].HostID + mapentries.Entries[i].MapRange
}
if offset+size <= d.state.OS.IdmapSet.Entries[0].HostID+d.state.OS.IdmapSet.Entries[0].MapRange {
set, err := mkIdmap(offset, size)
if err != nil && errors.Is(err, idmap.ErrHostIDIsSubID) {
return nil, 0, err
}
return set, offset, nil
}
return nil, 0, errors.New("Not enough uid/gid available for the container")
}
func (d *lxc) init() error {
// Compute the expanded config and device list
err := d.expandConfig()
if err != nil {
return err
}
return nil
}
func (d *lxc) initLXC(config bool) (*liblxc.Container, error) {
d.cMu.Lock()
defer d.cMu.Unlock()
// No need to go through all that for snapshots
if d.IsSnapshot() {
return nil, nil
}
// Check if being called from a hook
if d.fromHook {
return nil, errors.New("You can't use go-lxc from inside a LXC hook")
}
// Check if already initialized
if d.c != nil && (!config || d.cConfig) {
return d.c, nil
}
// As we are now going to be initialising a liblxc.Container reference, set the finalizer so that it is
// cleaned up (if needed) when the garbage collector destroys this instance struct.
d.cFinalizer.Do(func() { runtime.SetFinalizer(d, lxcUnload) })
reverter := revert.New()
defer reverter.Fail()
// Load the go-lxc struct
cname := project.Instance(d.Project().Name, d.Name())
cc, err := liblxc.NewContainer(cname, d.state.OS.LxcPath)
if err != nil {
return nil, err
}
reverter.Add(func() {
_ = cc.Release()
})
// Load cgroup abstraction
cg, err := d.cgroup(cc, false)
if err != nil {
return nil, err
}
// Setup logging
logfile := d.LogFilePath()
err = lxcSetConfigItem(cc, "lxc.log.file", logfile)
if err != nil {
return nil, err
}
logLevel := "warn"
if daemon.Debug {
logLevel = "trace"
} else if daemon.Verbose {
logLevel = "info"
}
err = lxcSetConfigItem(cc, "lxc.log.level", logLevel)
if err != nil {
return nil, err
}
if liblxc.RuntimeLiblxcVersionAtLeast(liblxc.Version(), 3, 0, 0) {
// Default size log buffer
err = lxcSetConfigItem(cc, "lxc.console.buffer.size", "auto")
if err != nil {
return nil, err
}
err = lxcSetConfigItem(cc, "lxc.console.size", "auto")
if err != nil {
return nil, err
}
// File to dump ringbuffer contents to when requested or
// container shutdown.
consoleBufferLogFile := d.ConsoleBufferLogPath()
err = lxcSetConfigItem(cc, "lxc.console.logfile", consoleBufferLogFile)
if err != nil {
return nil, err
}
}
if d.state.OS.CoreScheduling {
err = lxcSetConfigItem(cc, "lxc.sched.core", "1")
if err != nil {
return nil, err
}
}
// Allow for lightweight init
d.cConfig = config
if !config {
if d.c != nil {
_ = d.c.Release()
}
d.c = cc
reverter.Success()
return cc, err
}
if d.IsPrivileged() {
// Base config
toDrop := "sys_time sys_module sys_rawio"
if !d.state.OS.AppArmorStacking || d.state.OS.AppArmorStacked {
toDrop = toDrop + " mac_admin mac_override"
}
err = lxcSetConfigItem(cc, "lxc.cap.drop", toDrop)
if err != nil {
return nil, err
}
}
// Set an appropriate /proc, /sys/ and /sys/fs/cgroup
mounts := []string{}
if d.IsPrivileged() && !d.state.OS.RunningInUserNS {
mounts = append(mounts, "proc:mixed")
mounts = append(mounts, "sys:mixed")
} else {
mounts = append(mounts, "proc:rw")
mounts = append(mounts, "sys:rw")
}
mounts = append(mounts, "cgroup:rw:force")
err = lxcSetConfigItem(cc, "lxc.mount.auto", strings.Join(mounts, " "))
if err != nil {
return nil, err
}
err = lxcSetConfigItem(cc, "lxc.autodev", "1")
if err != nil {
return nil, err
}
err = lxcSetConfigItem(cc, "lxc.pty.max", "1024")
if err != nil {
return nil, err
}
bindMounts := []string{
"/dev/fuse",
"/dev/net/tun",
"/sys/firmware/efi/efivars",
"/sys/fs/fuse/connections",
"/sys/fs/pstore",
"/sys/kernel/config",
"/sys/kernel/debug",
"/sys/kernel/security",
"/sys/kernel/tracing",
}
// Handle unprivileged binfmt_misc.
if d.IsPrivileged() {
bindMounts = append(bindMounts, "/proc/sys/fs/binfmt_misc")
}
// Pass in /dev/zfs to the container if delegation is supported on the system.
// This is only done for unprivileged containers as delegation is tied to the user namespace.
if !d.IsPrivileged() && storageDrivers.ZFSSupportsDelegation() && util.PathExists("/dev/zfs") {
bindMounts = append(bindMounts, "/dev/zfs")
}
if d.IsPrivileged() && !d.state.OS.RunningInUserNS {
err = lxcSetConfigItem(cc, "lxc.mount.entry", "mqueue dev/mqueue mqueue rw,relatime,create=dir,optional 0 0")
if err != nil {
return nil, err
}
} else {
bindMounts = append(bindMounts, "/dev/mqueue")
}
for _, mnt := range bindMounts {
if !util.PathExists(mnt) {
continue
}
if internalUtil.IsDir(mnt) {
err = lxcSetConfigItem(cc, "lxc.mount.entry", fmt.Sprintf("%s %s none rbind,create=dir,optional 0 0", mnt, strings.TrimPrefix(mnt, "/")))
if err != nil {
return nil, err
}
} else {
err = lxcSetConfigItem(cc, "lxc.mount.entry", fmt.Sprintf("%s %s none bind,create=file,optional 0 0", mnt, strings.TrimPrefix(mnt, "/")))
if err != nil {
return nil, err
}
}
}
// For lxcfs
templateConfDir := os.Getenv("INCUS_LXC_TEMPLATE_CONFIG")
if templateConfDir == "" {
templateConfDir = "/usr/share/lxc/config"
}
if util.PathExists(fmt.Sprintf("%s/common.conf.d/", templateConfDir)) {
err = lxcSetConfigItem(cc, "lxc.include", fmt.Sprintf("%s/common.conf.d/", templateConfDir))
if err != nil {
return nil, err
}
}
// Configure devices cgroup
if d.IsPrivileged() && !d.state.OS.RunningInUserNS {
err = lxcSetConfigItem(cc, "lxc.cgroup2.devices.deny", "a")
if err != nil {
return nil, err
}
devices := []string{
"b *:* m", // Allow mknod of block devices
"c *:* m", // Allow mknod of char devices
"c 136:* rwm", // /dev/pts devices
"c 1:3 rwm", // /dev/null
"c 1:5 rwm", // /dev/zero
"c 1:7 rwm", // /dev/full
"c 1:8 rwm", // /dev/random
"c 1:9 rwm", // /dev/urandom
"c 5:0 rwm", // /dev/tty
"c 5:1 rwm", // /dev/console
"c 5:2 rwm", // /dev/ptmx
"c 10:229 rwm", // /dev/fuse
"c 10:200 rwm", // /dev/net/tun
}
if storageDrivers.ZFSSupportsDelegation() {
devices = append(devices, "c 10:249 rwm")
}
for _, dev := range devices {
err = lxcSetConfigItem(cc, "lxc.cgroup2.devices.allow", dev)
if err != nil {
return nil, err
}
}
}
if d.IsNesting() {
/*
* mount extra /proc and /sys to work around kernel
* restrictions on remounting them when covered
*/
err = lxcSetConfigItem(cc, "lxc.mount.entry", "proc dev/.lxc/proc proc create=dir,optional 0 0")
if err != nil {
return nil, err
}
err = lxcSetConfigItem(cc, "lxc.mount.entry", "sys dev/.lxc/sys sysfs create=dir,optional 0 0")
if err != nil {
return nil, err
}
}
// Setup architecture
personality, err := osarch.ArchitecturePersonality(d.architecture)
if err != nil {
personality, err = osarch.ArchitecturePersonality(d.state.OS.Architectures[0])
if err != nil {
return nil, err
}
}
err = lxcSetConfigItem(cc, "lxc.arch", personality)
if err != nil {
return nil, err
}
// Setup the hooks
err = lxcSetConfigItem(cc, "lxc.hook.version", "1")
if err != nil {
return nil, err
}
// Call the onstart hook on start.
err = lxcSetConfigItem(cc, "lxc.hook.pre-start", fmt.Sprintf("/proc/%d/exe callhook %s %s %s start", os.Getpid(), internalUtil.VarPath(""), util.SingleQuote(d.Project().Name), util.SingleQuote(d.Name())))
if err != nil {
return nil, err
}
// Call the onstopns hook on stop but before namespaces are unmounted.
err = lxcSetConfigItem(cc, "lxc.hook.stop", fmt.Sprintf("%s callhook %s %s %s stopns", d.state.OS.ExecPath, internalUtil.VarPath(""), util.SingleQuote(d.Project().Name), util.SingleQuote(d.Name())))
if err != nil {
return nil, err
}
// Call the onstop hook on stop.
err = lxcSetConfigItem(cc, "lxc.hook.post-stop", fmt.Sprintf("%s callhook %s %s %s stop", d.state.OS.ExecPath, internalUtil.VarPath(""), util.SingleQuote(d.Project().Name), util.SingleQuote(d.Name())))
if err != nil {
return nil, err
}
// Setup the console
err = lxcSetConfigItem(cc, "lxc.tty.max", "0")
if err != nil {
return nil, err
}
// Setup the hostname
err = lxcSetConfigItem(cc, "lxc.uts.name", d.Name())
if err != nil {
return nil, err
}
// Setup devIncus
if util.IsTrueOrEmpty(d.expandedConfig["security.guestapi"]) {
err = lxcSetConfigItem(cc, "lxc.mount.entry", fmt.Sprintf("%s dev/incus none bind,create=dir 0 0", internalUtil.VarPath("guestapi")))
if err != nil {
return nil, err
}
}
// Setup AppArmor
if d.state.OS.AppArmorAvailable {
if d.state.OS.AppArmorConfined || !d.state.OS.AppArmorAdmin {
// If confined but otherwise able to use AppArmor, use our own profile
curProfile := localUtil.AppArmorProfile()
curProfile = strings.TrimSuffix(curProfile, " (enforce)")
err := lxcSetConfigItem(cc, "lxc.apparmor.profile", curProfile)
if err != nil {
return nil, err
}
} else {
// If not currently confined, use the container's profile
profile := apparmor.InstanceProfileName(d)
/* In the nesting case, we want to enable the inside
* daemon to load its profile. Unprivileged containers can
* load profiles, but privileged containers cannot, so
* let's not use a namespace so they can fall back to
* the old way of nesting, i.e. using the parent's
* profile.
*/
if d.state.OS.AppArmorStacking && !d.state.OS.AppArmorStacked {
profile = fmt.Sprintf("%s//&:%s:", profile, apparmor.InstanceNamespaceName(d))
}
err := lxcSetConfigItem(cc, "lxc.apparmor.profile", profile)
if err != nil {
return nil, err
}
}
} else {
// Make sure that LXC won't try to apply an apparmor profile.
// This may fail on liblxc compiled without apparmor, so ignore errors.
_ = lxcSetConfigItem(cc, "lxc.apparmor.profile", "unconfined")
}
// Setup SELinux.
if d.state.OS.SELinuxEnabled {
selinuxContext := d.localConfig["volatile.selinux.context"]
if selinuxContext != "" {
logger.Debug("Setting SELinux context for container", logger.Ctx{"instance": d.Name(), "context": selinuxContext})
err = lxcSetConfigItem(cc, "lxc.selinux.context", selinuxContext)
if err != nil {
return nil, err
}
}
}
// Setup Seccomp if necessary
if seccomp.InstanceNeedsPolicy(d) {
err = lxcSetConfigItem(cc, "lxc.seccomp.profile", seccomp.ProfilePath(d))
if err != nil {
return nil, err
}
// Setup notification socket
// System requirement errors are handled during policy generation instead of here
ok, err := seccomp.InstanceNeedsIntercept(d.state, d)
if err == nil && ok {
err = lxcSetConfigItem(cc, "lxc.seccomp.notify.proxy", fmt.Sprintf("unix:%s", internalUtil.RunPath("seccomp.socket")))
if err != nil {
return nil, err
}
}
}
// Setup idmap
idmapset, err := d.NextIdmap()
if err != nil {
return nil, err
}
if idmapset != nil {
lines := idmapset.ToLXCString()
for _, line := range lines {
err := lxcSetConfigItem(cc, "lxc.idmap", line)
if err != nil {
return nil, err
}
}
}
// Setup environment
for k, v := range d.expandedConfig {
// gendoc:generate(entity=instance, group=miscellaneous, key=environment.*)
// The specified key/value environment variables are exported to the instance and set for `incus exec`.
// ---
// type: string
// liveupdate: yes (exec)
// shortdesc: Environment variables to export
after, ok := strings.CutPrefix(k, "environment.")
if ok {
// LXC supports quoting the value between " even if the value itself contains ".
err = lxcSetConfigItem(cc, "lxc.environment", fmt.Sprintf("\"%s=%s\"", after, v))
if err != nil {
return nil, err
}
}
}
err = lxcSetConfigItem(cc, "lxc.environment", "CREDENTIALS_DIRECTORY=/dev/.incus-systemd-credentials")
if err != nil {
return nil, err
}
err = lxcSetConfigItem(cc, "lxc.mount.entry", fmt.Sprintf("%s dev/.incus-systemd-credentials none bind,ro,create=dir 0 0", filepath.Join(d.Path(), "credentials")))
if err != nil {
return nil, err
}
// Setup NVIDIA runtime
if util.IsTrue(d.expandedConfig["nvidia.runtime"]) {
hookDir := os.Getenv("INCUS_LXC_HOOK")
if hookDir == "" {
hookDir = "/usr/share/lxc/hooks"
}
hookPath := filepath.Join(hookDir, "nvidia")
if !util.PathExists(hookPath) {
return nil, errors.New("The NVIDIA LXC hook couldn't be found")
}
_, err := exec.LookPath("nvidia-container-cli")
if err != nil {
return nil, errors.New("The NVIDIA container tools couldn't be found")
}
err = lxcSetConfigItem(cc, "lxc.environment", "NVIDIA_VISIBLE_DEVICES=none")
if err != nil {
return nil, err
}
nvidiaDriver := d.expandedConfig["nvidia.driver.capabilities"]
if nvidiaDriver == "" {
err = lxcSetConfigItem(cc, "lxc.environment", "NVIDIA_DRIVER_CAPABILITIES=compute,utility")
if err != nil {
return nil, err
}
} else {
err = lxcSetConfigItem(cc, "lxc.environment", fmt.Sprintf("\"NVIDIA_DRIVER_CAPABILITIES=%s\"", nvidiaDriver))
if err != nil {
return nil, err
}
}
nvidiaRequireCuda := d.expandedConfig["nvidia.require.cuda"]
if nvidiaRequireCuda != "" {
err = lxcSetConfigItem(cc, "lxc.environment", fmt.Sprintf("\"NVIDIA_REQUIRE_CUDA=%s\"", nvidiaRequireCuda))
if err != nil {
return nil, err
}
}
nvidiaRequireDriver := d.expandedConfig["nvidia.require.driver"]
if nvidiaRequireDriver != "" {
err = lxcSetConfigItem(cc, "lxc.environment", fmt.Sprintf("\"NVIDIA_REQUIRE_DRIVER=%s\"", nvidiaRequireDriver))
if err != nil {
return nil, err
}
}
err = lxcSetConfigItem(cc, "lxc.hook.mount", hookPath)
if err != nil {
return nil, err
}
}
// Memory limits
if cgroup.Supports(cgroup.Memory) {
memory := d.expandedConfig["limits.memory"]
memoryEnforce := d.expandedConfig["limits.memory.enforce"]
memorySwap := d.expandedConfig["limits.memory.swap"]
memorySwapPriority := d.expandedConfig["limits.memory.swap.priority"]
// Configure the memory limits
if memory != "" {
valueInt, err := ParseMemoryStr(memory)
if err != nil {
return nil, err
}
if memoryEnforce == "soft" {
err = cg.SetMemorySoftLimit(valueInt)
if err != nil {
return nil, err
}
} else {
err = cg.SetMemoryLimit(valueInt)
if err != nil {
return nil, err
}
if util.IsTrueOrEmpty(memorySwap) || util.IsFalse(memorySwap) {
err = cg.SetMemorySwapLimit(0)
if err != nil {
// Ignore missing swap accounting unless explicitly configured.
if memorySwap != "" || !errors.Is(err, cgroup.ErrControllerMissing) {
return nil, err
}
}
} else {
// Additional memory as swap.
swapInt, err := units.ParseByteSizeString(memorySwap)
if err != nil {
return nil, err
}
err = cg.SetMemorySwapLimit(swapInt)
if err != nil {
return nil, err
}
}
}
}
// Configure the swappiness
if util.IsFalse(memorySwap) {
err = cg.SetMemorySwappiness(0)
if err != nil && !errors.Is(err, cgroup.ErrControllerMissing) {
return nil, err
}
} else if memorySwapPriority != "" {
priority, err := strconv.Atoi(memorySwapPriority)
if err != nil {
return nil, err
}
// Maximum priority (10) should be default swappiness (60).
err = cg.SetMemorySwappiness(int64(70 - priority))
if err != nil {
return nil, err
}
}
}
// CPU limits
cpuPriority := d.expandedConfig["limits.cpu.priority"]
cpuAllowance := d.expandedConfig["limits.cpu.allowance"]
if (cpuPriority != "" || cpuAllowance != "") && cgroup.Supports(cgroup.CPU) {
cpuShares, cpuCfsQuota, cpuCfsPeriod, err := cgroup.ParseCPU(cpuAllowance, cpuPriority)
if err != nil {
return nil, err
}
if cpuShares != 1024 {
err = cg.SetCPUShare(cpuShares)
if err != nil {
return nil, err
}
}
if cpuCfsPeriod != -1 && cpuCfsQuota != -1 {
err = cg.SetCPUCfsLimit(cpuCfsPeriod, cpuCfsQuota)
if err != nil {
return nil, err
}
}
}
// Disk priority limits.
diskPriority := d.ExpandedConfig()["limits.disk.priority"]
if diskPriority != "" {
if !cgroup.Supports(cgroup.IO) {
return nil, errors.New("Cannot apply limits.disk.priority as blkio.weight cgroup controller is missing")
}
priorityInt, err := strconv.Atoi(diskPriority)
if err != nil {
return nil, err
}
priority := priorityInt * 100
// Minimum valid value is 10
if priority == 0 {
priority = 10
}
err = cg.SetBlkioWeight(int64(priority))
if err != nil {
return nil, err
}
}
// Processes
if cgroup.Supports(cgroup.Pids) {
processes := d.expandedConfig["limits.processes"]
if processes != "" {
valueInt, err := strconv.ParseInt(processes, 10, 64)
if err != nil {
return nil, err
}
err = cg.SetMaxProcesses(valueInt)
if err != nil {
return nil, err
}
}
}
// Hugepages
if cgroup.Supports(cgroup.Hugetlb) {
for i, key := range internalInstance.HugePageSizeKeys {
value := d.expandedConfig[key]
if value != "" {
value, err := units.ParseByteSizeString(value)
if err != nil {
return nil, err
}
err = cg.SetHugepagesLimit(internalInstance.HugePageSizeSuffix[i], value)
if err != nil {
return nil, err
}
}
}
}
// Setup process limits
for k, v := range d.expandedConfig {
after, ok := strings.CutPrefix(k, "limits.kernel.")
if ok {
prlimitSuffix := after
prlimitKey := fmt.Sprintf("lxc.prlimit.%s", prlimitSuffix)
err = lxcSetConfigItem(cc, prlimitKey, v)
if err != nil {
return nil, err
}
}
}
// Setup sysctls
for k, v := range d.expandedConfig {
// gendoc:generate(entity=instance, group=miscellaneous, key=linux.sysctl.*)
//
// ---
// type: string
// liveupdate: no
// condition: container
// shortdesc: Override for the corresponding `sysctl` setting in the container
after, ok := strings.CutPrefix(k, "linux.sysctl.")
if ok {
sysctlSuffix := after
sysctlKey := fmt.Sprintf("lxc.sysctl.%s", sysctlSuffix)
err = lxcSetConfigItem(cc, sysctlKey, v)
if err != nil {
return nil, err
}
}
}
// Setup shmounts
err = lxcSetConfigItem(cc, "lxc.mount.auto", fmt.Sprintf("shmounts:%s:/dev/.incus-mounts", d.ShmountsPath()))
if err != nil {
return nil, err
}
if d.c != nil {
_ = d.c.Release()
}
d.c = cc
reverter.Success()
return cc, err
}
var (
idmappedStorageMap map[unix.Fsid]idmap.StorageType = map[unix.Fsid]idmap.StorageType{}
idmappedStorageMapString map[string]idmap.StorageType = map[string]idmap.StorageType{}
idmappedStorageMapLock sync.Mutex
)
// IdmappedStorage determines if the container can use idmapped mounts.
func (d *lxc) IdmappedStorage(fspath string, fstype string) idmap.StorageType {
var mode idmap.StorageType = idmap.StorageTypeNone
bindMount := fstype == "none" || fstype == ""
buf := &unix.Statfs_t{}
if bindMount {
err := unix.Statfs(fspath, buf)
if err != nil {
d.logger.Error("Failed to statfs", logger.Ctx{"path": fspath, "err": err})
return mode
}
}
idmappedStorageMapLock.Lock()
defer idmappedStorageMapLock.Unlock()
if bindMount {
val, ok := idmappedStorageMap[buf.Fsid]
if ok {
// Return recorded idmapping type.
return val
}
} else {
val, ok := idmappedStorageMapString[fstype]
if ok {
// Return recorded idmapping type.
return val
}
}
if idmap.CanIdmapMount(fspath, fstype) {
// Use idmapped mounts.
mode = idmap.StorageTypeIdmapped
}
if bindMount {
idmappedStorageMap[buf.Fsid] = mode
} else {
idmappedStorageMapString[fstype] = mode
}
return mode
}
func (d *lxc) devIncusEventSend(eventType string, eventMessage map[string]any) error {
event := jmap.Map{}
event["type"] = eventType
event["timestamp"] = time.Now()
event["metadata"] = eventMessage
return d.state.DevIncusEvents.Send(d.ID(), eventType, eventMessage)
}
// RegisterDevices calls the Register() function on all of the instance's devices.
func (d *lxc) RegisterDevices() {
d.devicesRegister(d)
}
// deviceStart loads a new device and calls its Start() function.
func (d *lxc) deviceStart(dev device.Device, instanceRunning bool) (*deviceConfig.RunConfig, error) {
configCopy := dev.Config()
l := d.logger.AddContext(logger.Ctx{"device": dev.Name(), "type": configCopy["type"]})
l.Debug("Starting device")
reverter := revert.New()
defer reverter.Fail()
if instanceRunning && !dev.CanHotPlug() {
return nil, errors.New("Device cannot be started when instance is running")
}
runConf, err := dev.Start()
if err != nil {
return nil, err
}
reverter.Add(func() {
runConf, _ := dev.Stop()
if runConf != nil {
_ = d.runHooks(runConf.PostHooks)
}
})
// If runConf supplied, perform any container specific setup of device.
if runConf != nil {
// Shift device file ownership if needed before mounting into container.
// This needs to be done whether or not container is running.
if len(runConf.Mounts) > 0 {
err := d.deviceStaticShiftMounts(runConf.Mounts)
if err != nil {
return nil, err
}
}
// If container is running and then live attach device.
if instanceRunning {
// Attach mounts if requested.
if len(runConf.Mounts) > 0 {
err = d.deviceHandleMounts(runConf.Mounts)
if err != nil {
return nil, err
}
}
// Add cgroup rules if requested.
if len(runConf.CGroups) > 0 {
err = d.deviceAddCgroupRules(runConf.CGroups)
if err != nil {
return nil, err
}
}
// Attach network interface if requested.
if len(runConf.NetworkInterface) > 0 {
err = d.deviceAttachNIC(dev.Name(), configCopy, runConf.NetworkInterface)
if err != nil {
return nil, err
}
}
// If running, run post start hooks now (if not running, they will be run
// once the instance is started).
err = d.runHooks(runConf.PostHooks)
if err != nil {
return nil, err
}
}
}
reverter.Success()
return runConf, nil
}
// deviceStaticShiftMounts statically shift device mount files ownership to active idmap if needed.
func (d *lxc) deviceStaticShiftMounts(mounts []deviceConfig.MountEntryItem) error {
idmapSet, err := d.CurrentIdmap()
if err != nil {
return fmt.Errorf("Failed to get idmap for device: %s", err)
}
// If there is an idmap being applied and the daemon is not running in a user namespace then shift the
// device files before they are mounted.
if idmapSet != nil && !d.state.OS.RunningInUserNS {
for _, mount := range mounts {
// Skip UID/GID shifting if OwnerShift mode is not static, or the host-side
// DevPath is empty (meaning an unmount request that doesn't need shifting).
if mount.OwnerShift != deviceConfig.MountOwnerShiftStatic || mount.DevPath == "" {
continue
}
err := idmapSet.ShiftPath(mount.DevPath, nil)
if err != nil {
// uidshift failing is weird, but not a big problem. Log and proceed.
d.logger.Debug("Failed to uidshift device", logger.Ctx{"mountDevPath": mount.DevPath, "err": err})
}
}
}
return nil
}
// deviceAddCgroupRules live adds cgroup rules to a container.
func (d *lxc) deviceAddCgroupRules(cgroups []deviceConfig.RunConfigItem) error {
_, err := d.initLXC(false)
if err != nil {
return err
}
for _, rule := range cgroups {
// Only apply devices cgroup rules if container is running privileged and host has devices cgroup controller.
if strings.HasPrefix(rule.Key, "devices.") && (!d.isCurrentlyPrivileged() || d.state.OS.RunningInUserNS) {
continue
}
// Add the new device cgroup rule.
err := d.CGroupSet(rule.Key, rule.Value)
if err != nil {
return fmt.Errorf("Failed to add cgroup rule for device: %w", err)
}
}
return nil
}
// deviceAttachNIC live attaches a NIC device to a container.
func (d *lxc) deviceAttachNIC(devName string, configCopy map[string]string, netIF []deviceConfig.RunConfigItem) error {
ctDevName := ""
connected := true
for _, dev := range netIF {
switch dev.Key {
case "link":
ctDevName = dev.Value
case "connected":
connected = util.IsTrueOrEmpty(dev.Value)
}
}
if ctDevName == "" {
return errors.New("Device didn't provide a link property to use")
}
// Load the go-lxc struct.
cc, err := d.initLXC(false)
if err != nil {
return err
}
// Add the interface to the container.
err = cc.AttachInterface(ctDevName, configCopy["name"])
if err != nil {
return fmt.Errorf("Failed to attach interface: %s to %s: %w", ctDevName, configCopy["name"], err)
}
return d.setNICLink(devName, connected, true)
}
// deviceStop loads a new device and calls its Stop() function.
// Accepts a stopHookNetnsPath argument which is required when run from the onStopNS hook before the
// container's network namespace is unmounted (which is required for NIC device cleanup).
func (d *lxc) deviceStop(dev device.Device, instanceRunning bool, stopHookNetnsPath string) error {
configCopy := dev.Config()
l := d.logger.AddContext(logger.Ctx{"device": dev.Name(), "type": configCopy["type"]})
l.Debug("Stopping device")
if instanceRunning && !dev.CanHotPlug() {
return errors.New("Device cannot be stopped when instance is running")
}
runConf, err := dev.Stop()
if err != nil {
return err
}
if runConf != nil {
// If network interface settings returned, then detach NIC from container.
if len(runConf.NetworkInterface) > 0 {
err = d.deviceDetachNIC(configCopy, runConf.NetworkInterface, instanceRunning, stopHookNetnsPath)
if err != nil {
return err
}
}
// Add cgroup rules if requested and container is running.
if len(runConf.CGroups) > 0 && instanceRunning {
err = d.deviceAddCgroupRules(runConf.CGroups)
if err != nil {
return err
}
}
// Detach mounts if requested and container is running.
if len(runConf.Mounts) > 0 && instanceRunning {
err = d.deviceHandleMounts(runConf.Mounts)
if err != nil {
return err
}
}
// Run post stop hooks irrespective of run state of instance.
err = d.runHooks(runConf.PostHooks)
if err != nil {
return err
}
}
return nil
}
// deviceDetachNIC detaches a NIC device from a container.
// Accepts a stopHookNetnsPath argument which is required when run from the onStopNS hook before the
// container's network namespace is unmounted (which is required for NIC device cleanup).
func (d *lxc) deviceDetachNIC(configCopy map[string]string, netIF []deviceConfig.RunConfigItem, instanceRunning bool, stopHookNetnsPath string) error {
// Get requested device name to detach interface back to on the host.
devName := ""
for _, dev := range netIF {
if dev.Key == "link" {
devName = dev.Value
break
}
}
if devName == "" {
return errors.New("Device didn't provide a link property to use")
}
// If container is running, perform live detach of interface back to host.
if instanceRunning {
// For some reason, having network config confuses detach, so get our own go-lxc struct.
cname := project.Instance(d.Project().Name, d.Name())
cc, err := liblxc.NewContainer(cname, d.state.OS.LxcPath)
if err != nil {
return err
}
defer logger.WarnOnError(cc.Release, "Failed to release container")
// Get interfaces inside container.
ifaces, err := cc.Interfaces()
if err != nil {
return fmt.Errorf("Failed to list network interfaces: %w", err)
}
// If interface doesn't exist inside container, cannot proceed.
if !slices.Contains(ifaces, configCopy["name"]) {
return nil
}
err = cc.DetachInterfaceRename(configCopy["name"], devName)
if err != nil {
return fmt.Errorf("Failed to detach interface: %q to %q: %w", configCopy["name"], devName, err)
}
} else {
// Currently liblxc does not move devices back to the host on stop that were added
// after the container was started. For this reason we utilise the lxc.hook.stop
// hook so that we can capture the netns path, enter the namespace and move the nics
// back to the host and rename them if liblxc hasn't already done it.
// We can only move back devices that have an expected host_name record and where
// that device doesn't already exist on the host as if a device exists on the host
// we can't know whether that is because liblxc has moved it back already or whether
// it is a conflicting device.
if !util.PathExists(fmt.Sprintf("/sys/class/net/%s", devName)) {
if stopHookNetnsPath == "" {
return fmt.Errorf("Cannot detach NIC device %q without stopHookNetnsPath being provided", devName)
}
err := d.detachInterfaceRename(stopHookNetnsPath, configCopy["name"], devName)
if err != nil {
return fmt.Errorf("Failed to detach interface: %q to %q: %w", configCopy["name"], devName, err)
}
d.logger.Debug("Detached NIC device interface", logger.Ctx{"name": configCopy["name"], "device": devName})
}
}
return nil
}
// deviceHandleMounts live attaches or detaches mounts on a container.
// If the mount DevPath is empty the mount action is treated as unmount.
func (d *lxc) deviceHandleMounts(mounts []deviceConfig.MountEntryItem) error {
for _, mount := range mounts {
if mount.DevPath != "" {
flags := 0
// Convert options into flags.
for _, opt := range mount.Opts {
switch opt {
case "bind":
flags |= unix.MS_BIND
case "rbind":
flags |= unix.MS_BIND | unix.MS_REC
case "ro":
flags |= unix.MS_RDONLY
}
}
var idmapType idmap.StorageType = idmap.StorageTypeNone
if !d.IsPrivileged() && mount.OwnerShift == deviceConfig.MountOwnerShiftDynamic {
idmapType = d.IdmappedStorage(mount.DevPath, mount.FSType)
if idmapType == idmap.StorageTypeNone {
return errors.New("Required idmapping abilities not available")
}
}
// Mount it into the container.
err := d.insertMount(mount.DevPath, mount.TargetPath, mount.FSType, flags, idmapType)
if err != nil {
return fmt.Errorf("Failed to add mount for device inside container: %s", err)
}
} else {
err := func() error {
relativeTargetPath := strings.TrimPrefix(mount.TargetPath, "/")
// Connect to files API.
files, err := d.FileSFTP()
if err != nil {
return err
}
defer logger.WarnOnError(files.Close, "Failed to close SFTP connection")
_, err = files.Lstat(relativeTargetPath)
if err == nil {
err := d.removeMount(mount.TargetPath)
if err != nil {
return fmt.Errorf("Error unmounting the device path inside container: %s", err)
}
// Only remove mountpoints created in /dev.
if strings.HasPrefix(mount.TargetPath, "dev/") {
err := files.Remove(relativeTargetPath)
if err != nil {
return err
}
}
}
return nil
}()
if err != nil {
return err
}
}
}
return nil
}
// DeviceEventHandler actions the results of a RunConfig after an event has occurred on a device.
func (d *lxc) DeviceEventHandler(runConf *deviceConfig.RunConfig) error {
// Device events can only be processed when the container is running.
// We use InitPID here rather than IsRunning because this task can be triggered during the
// container startup process, which is during the time that the start lock is held, which causes
// IsRunning to return false (because the container hasn't fully started yet).
if d.InitPID() <= 0 {
return nil
}
if runConf == nil {
return nil
}
// Shift device file ownership if needed before mounting devices into container.
if len(runConf.Mounts) > 0 {
err := d.deviceStaticShiftMounts(runConf.Mounts)
if err != nil {
return err
}
err = d.deviceHandleMounts(runConf.Mounts)
if err != nil {
return err
}
}
// Add cgroup rules if requested.
if len(runConf.CGroups) > 0 {
err := d.deviceAddCgroupRules(runConf.CGroups)
if err != nil {
return err
}
}
// Handle NIC reconfiguration.
var devName string
var connected bool
for _, dev := range runConf.NetworkInterface {
switch dev.Key {
case "devName":
devName = dev.Value
case "connected":
connected = util.IsTrueOrEmpty(dev.Value)
}
}
if devName != "" {
err := d.setNICLink(devName, connected, false)
if err != nil {
return err
}
}
// Run any post hooks requested.
err := d.runHooks(runConf.PostHooks)
if err != nil {
return err
}
// Generate uevent inside container if requested.
if len(runConf.Uevents) > 0 {
pidFd, err := d.InitPidFd()
if err != nil {
return err
}
defer logger.WarnOnError(pidFd.Close, "Failed to close PID fd")
for _, eventParts := range runConf.Uevents {
length := 0
for _, part := range eventParts {
length = length + len(part) + 1
}
args := []string{
"forkuevent",
"inject",
"--",
fmt.Sprintf("%d", d.InitPID()),
"3",
fmt.Sprintf("%d", length),
}
args = append(args, eventParts...)
_, _, err = subprocess.RunCommandSplit(
context.TODO(),
nil,
[]*os.File{pidFd},
d.state.OS.ExecPath,
args...,
)
if err != nil {
return err
}
}
}
return nil
}
func (d *lxc) handleIdmappedStorage() (idmap.StorageType, *idmap.Set, error) {
diskIdmap, err := d.DiskIdmap()
if err != nil {
return idmap.StorageTypeNone, nil, fmt.Errorf("Set last ID map: %w", err)
}
nextIdmap, err := d.NextIdmap()
if err != nil {
return idmap.StorageTypeNone, nil, fmt.Errorf("Set ID map: %w", err)
}
// Identical on-disk idmaps so no changes required.
if nextIdmap.Equals(diskIdmap) {
return idmap.StorageTypeNone, nextIdmap, nil
}
// There's no on-disk idmap applied and the container can use idmapped
// storage.
idmapType := d.IdmappedStorage(d.RootfsPath(), "none")
if diskIdmap == nil && idmapType != idmap.StorageTypeNone {
return idmapType, nextIdmap, nil
}
// We need to change the on-disk idmap but the container is protected
// against idmap changes.
if util.IsTrue(d.expandedConfig["security.protection.shift"]) {
return idmap.StorageTypeNone, nil, errors.New("Container is protected against filesystem shifting")
}
d.logger.Debug("Container idmap changed, remapping")
d.updateProgress("Remapping container filesystem")
storageType, err := d.getStorageType()
if err != nil {
return idmap.StorageTypeNone, nil, fmt.Errorf("Storage type: %w", err)
}
// Revert the currently applied on-disk idmap.
if diskIdmap != nil {
switch storageType {
case "zfs":
err = diskIdmap.UnshiftPath(d.RootfsPath(), storageDrivers.ShiftZFSSkipper)
case "btrfs":
err = storageDrivers.UnshiftBtrfsRootfs(d.RootfsPath(), diskIdmap)
default:
err = diskIdmap.UnshiftPath(d.RootfsPath(), nil)
}
if err != nil {
return idmap.StorageTypeNone, nil, err
}
}
jsonDiskIdmap := "[]"
// If the container can't use idmapped storage apply the new on-disk
// idmap of the container now. Otherwise we will later instruct LXC to
// make use of idmapped storage.
if nextIdmap != nil && idmapType == idmap.StorageTypeNone {
switch storageType {
case "zfs":
err = nextIdmap.ShiftPath(d.RootfsPath(), storageDrivers.ShiftZFSSkipper)
case "btrfs":
err = storageDrivers.ShiftBtrfsRootfs(d.RootfsPath(), nextIdmap)
default:
err = nextIdmap.ShiftPath(d.RootfsPath(), nil)
}
if err != nil {
return idmap.StorageTypeNone, nil, err
}
idmapJSON, err := nextIdmap.ToJSON()
if err != nil {
return idmap.StorageTypeNone, nil, err
}
jsonDiskIdmap = idmapJSON
}
err = d.VolatileSet(map[string]string{"volatile.last_state.idmap": jsonDiskIdmap})
if err != nil {
return idmap.StorageTypeNone, nextIdmap, fmt.Errorf("Set volatile.last_state.idmap config key on container %q (id %d): %w", d.name, d.id, err)
}
d.updateProgress("")
return idmapType, nextIdmap, nil
}
// selinuxEnsureContext generates and persists the SELinux context for this instance.
// Returns true if we need to relabel the rootfs, false if labeling is not required or wanted.
func (d *lxc) selinuxEnsureContext() (bool, error) {
if !d.state.OS.SELinuxEnabled {
return false, nil
}
previousCtx := d.localConfig["volatile.selinux.context"]
allocLevel := func() (string, func(), error) {
used, err := d.selinuxCollectUsedLevels()
if err != nil {
return "", nil, err
}
return selinux.AllocateLevel(used)
}
ctx, needsPersist, release, err := selinux.InstanceContext(d.state.OS, instancetype.Container, d.localConfig, d.expandedConfig, allocLevel)
if err != nil {
return false, err
}
defer release()
if ctx == "" {
return false, nil
}
if needsPersist {
err = d.VolatileSet(map[string]string{"volatile.selinux.context": ctx})
if err != nil {
return false, fmt.Errorf("Failed to persist SELinux context: %w", err)
}
}
// Return true if this is the first time a context was generated.
return previousCtx == "", nil
}
// selinuxLabelFiles applies SELinux file labels to the instance rootfs.
func (d *lxc) selinuxLabelFiles(contextIsNew bool) error {
if !d.state.OS.SELinuxEnabled {
return nil
}
ctx := d.localConfig["volatile.selinux.context"]
if ctx == "" {
return nil
}
skipPath := ""
rootfsMode := d.expandedConfig["security.selinux.label_rootfs"]
if rootfsMode == "" {
rootfsMode = "auto"
}
logger.Debug("SELinux label mode", logger.Ctx{"mode": rootfsMode})
switch rootfsMode {
case "auto":
// Skip re-labeling if not first start and level is explicitly set.
if !contextIsNew && d.localConfig["security.selinux.level"] != "" {
skipPath = d.RootfsPath()
}
case "never":
skipPath = d.RootfsPath()
case "always":
// Always relabel on every start.
default:
return fmt.Errorf("Invalid security.selinux.label_rootfs value: %q", rootfsMode)
}
fileCtx := selinux.InstanceFileContext(ctx, instancetype.Container, d.expandedConfig)
if fileCtx == "" {
return fmt.Errorf("Failed to derive file context from %q", ctx)
}
return selinux.LabelTree(d.Path(), fileCtx, skipPath)
}
// Start functions.
func (d *lxc) startCommon() (string, []func() error, error) {
postStartHooks := []func() error{}
reverter := revert.New()
defer reverter.Fail()
// Assign NUMA node(s) if needed.
if d.expandedConfig["limits.cpu.nodes"] == "balanced" {
err := d.balanceNUMANodes()
if err != nil {
return "", nil, err
}
}
// Check if idmap needs changing.
if !d.IsPrivileged() {
nextMap, err := d.NextIdmap()
if err != nil {
return "", nil, err
}
// Check if we need to change idmap.
if nextMap != nil && d.state.OS.IdmapSet != nil && !d.state.OS.IdmapSet.Includes(nextMap) {
// Update the idmap.
idmapSet, base, err := d.findIdmap()
if err != nil {
return "", nil, fmt.Errorf("Failed to get ID map: %w", err)
}
idmapSetJSON, err := idmapSet.ToJSON()
if err != nil {
return "", nil, fmt.Errorf("Failed to encode ID map: %w", err)
}
err = d.VolatileSet(map[string]string{
"volatile.idmap.next": idmapSetJSON,
"volatile.idmap.base": fmt.Sprintf("%v", base),
})
if err != nil {
return "", nil, fmt.Errorf("Failed to update volatile idmap: %w", err)
}
// Invalidate the idmap cache.
d.idmapset = nil
}
}
// Ensure SELinux context is generated and persisted.
contextIsNew, err := d.selinuxEnsureContext()
if err != nil {
return "", nil, err
}
// Load the go-lxc struct
cc, err := d.initLXC(true)
if err != nil {
return "", nil, fmt.Errorf("Load go-lxc struct: %w", err)
}
// gendoc:generate(entity=image, group=requirements, key=requirements.cgroup)
//
// ---
// type: string
// shortdesc: If set to `v1`, indicates that the image requires the host to run cgroup v1.
//
// Ensure cgroup v1 configuration is set appropriately with the image using systemd
if d.localConfig["image.requirements.cgroup"] == "v1" && !util.PathExists("/sys/fs/cgroup/systemd") {
return "", nil, errors.New("The image used by this instance requires a CGroupV1 host system")
}
// gendoc:generate(entity=image, group=requirements, key=requirements.privileged)
//
// ---
// type: bool
// shortdesc: If set to `false`, indicates that the image cannot work as a privileged container.
//
// Ensure privileged is turned off for images that cannot work privileged
if util.IsFalse(d.localConfig["image.requirements.privileged"]) && util.IsTrue(d.expandedConfig["security.privileged"]) {
return "", nil, errors.New("The image used by this instance is incompatible with privileged containers. Please unset security.privileged on the instance")
}
// Load any required kernel modules
kernelModules := d.expandedConfig["linux.kernel_modules"]
if kernelModules != "" {
for _, module := range strings.Split(kernelModules, ",") {
module = strings.TrimPrefix(module, " ")
err := linux.LoadModule(module)
if err != nil {
return "", nil, fmt.Errorf("Failed to load kernel module '%s': %w", module, err)
}
}
}
// Rotate the log file.
logfile := d.LogFilePath()
if util.PathExists(logfile) {
_ = os.Remove(logfile + ".old")
err := os.Rename(logfile, logfile+".old")
if err != nil && !errors.Is(err, fs.ErrNotExist) {
return "", nil, err
}
}
// Wait for any file operations to complete.
// This is to avoid having an active mount by forkfile and so all file operations
// from this point will use the container's namespace rather than a chroot.
d.stopForkfile(false)
// Mount instance root volume.
mountInfo, err := d.mount()
if err != nil {
return "", nil, err
}
// Handle post hooks.
postStartHooks = append(postStartHooks, func() error {
for _, hook := range mountInfo.PostHooks {
err := hook(d)
if err != nil {
return err
}
}
return nil
})
reverter.Add(func() { _ = d.unmount() })
idmapType, nextIdmap, err := d.handleIdmappedStorage()
if err != nil {
return "", nil, fmt.Errorf("Failed to handle idmapped storage: %w", err)
}
nextIdmapJSON, err := nextIdmap.ToJSON()
if err != nil {
return "", nil, fmt.Errorf("Failed to encode ID map: %w", err)
}
if d.localConfig["volatile.idmap.current"] != nextIdmapJSON {
err = d.VolatileSet(map[string]string{"volatile.idmap.current": nextIdmapJSON})
if err != nil {
return "", nil, fmt.Errorf("Set volatile.idmap.current config key on container %q (id %d): %w", d.name, d.id, err)
}
}
// Generate the Seccomp profile
err = seccomp.CreateProfile(d.state, d)
if err != nil {
return "", nil, err
}
// Cleanup any existing leftover devices
_ = d.removeUnixDevices()
_ = d.removeDiskDevices()
// Create any missing directories.
err = os.MkdirAll(d.LogPath(), 0o700)
if err != nil {
return "", nil, err
}
err = os.MkdirAll(d.RunPath(), 0o700)
if err != nil {
return "", nil, err
}
err = os.MkdirAll(d.DevicesPath(), 0o711)
if err != nil {
return "", nil, err
}
err = os.MkdirAll(d.ShmountsPath(), 0o711)
if err != nil {
return "", nil, err
}
volatileSet := make(map[string]string)
// Generate UUID if not present (do this before UpdateBackupFile() call).
instUUID := d.localConfig["volatile.uuid"]
if instUUID == "" {
instUUID = uuid.New().String()
volatileSet["volatile.uuid"] = instUUID
}
// For a container instance, we must also set the generation UUID.
genUUID := d.localConfig["volatile.uuid.generation"]
if genUUID == "" {
genUUID = instUUID
volatileSet["volatile.uuid.generation"] = genUUID
}
// Create the devices
nicID := -1
nvidiaDevices := []string{}
sortedDevices := d.expandedDevices.Sorted()
startDevices := make([]device.Device, 0, len(sortedDevices))
// Load devices in sorted order, this ensures that device mounts are added in path order.
// Loading all devices first means that validation of all devices occurs before starting any of them.
for _, entry := range sortedDevices {
dev, err := d.deviceLoad(d, entry.Name, entry.Config, false)
if err != nil {
if errors.Is(err, device.ErrUnsupportedDevType) {
continue // Skip unsupported device (allows for mixed instance type profiles).
}
return "", nil, fmt.Errorf("Failed start validation for device %q: %w", entry.Name, err)
}
// Run pre-start of check all devices before starting any device to avoid expensive revert.
err = dev.PreStartCheck()
if err != nil {
return "", nil, fmt.Errorf("Failed pre-start check for device %q: %w", dev.Name(), err)
}
startDevices = append(startDevices, dev)
}
// Start devices in order.
for i := range startDevices {
dev := startDevices[i] // Local var for revert.
// Start the device.
runConf, err := d.deviceStart(dev, false)
if err != nil {
return "", nil, fmt.Errorf("Failed to start device %q: %w", dev.Name(), err)
}
// Stop device on failure to setup container.
reverter.Add(func() {
err := d.deviceStop(dev, false, "")
if err != nil {
d.logger.Error("Failed to cleanup device", logger.Ctx{"device": dev.Name(), "err": err})
}
})
if runConf == nil {
continue
}
if runConf.Revert != nil {
reverter.Add(runConf.Revert)
}
// Process rootfs setup.
if runConf.RootFS.Path != "" {
if !liblxc.RuntimeLiblxcVersionAtLeast(liblxc.Version(), 2, 1, 0) {
// Set the rootfs backend type if supported (must happen before any other lxc.rootfs)
err := lxcSetConfigItem(cc, "lxc.rootfs.backend", "dir")
if err == nil {
value := cc.ConfigItem("lxc.rootfs.backend")
if len(value) == 0 || value[0] != "dir" {
_ = lxcSetConfigItem(cc, "lxc.rootfs.backend", "")
}
}
}
// Get an absolute path for the rootfs (avoid constantly traversing the symlink).
absoluteRootfs, err := filepath.EvalSymlinks(runConf.RootFS.Path)
if err != nil {
return "", nil, fmt.Errorf("Unable to resolve container rootfs: %w", err)
}
if liblxc.RuntimeLiblxcVersionAtLeast(liblxc.Version(), 2, 1, 0) {
rootfsPath := fmt.Sprintf("dir:%s", absoluteRootfs)
err = lxcSetConfigItem(cc, "lxc.rootfs.path", rootfsPath)
} else {
err = lxcSetConfigItem(cc, "lxc.rootfs", absoluteRootfs)
}
if err != nil {
return "", nil, fmt.Errorf("Failed to setup device rootfs %q: %w", dev.Name(), err)
}
if len(runConf.RootFS.Opts) > 0 {
err = lxcSetConfigItem(cc, "lxc.rootfs.options", strings.Join(runConf.RootFS.Opts, ","))
if err != nil {
return "", nil, fmt.Errorf("Failed to setup device rootfs %q: %w", dev.Name(), err)
}
}
if !d.IsPrivileged() && idmapType == idmap.StorageTypeIdmapped {
err = lxcSetConfigItem(cc, "lxc.rootfs.options", "idmap=container")
if err != nil {
return "", nil, fmt.Errorf("Failed to set \"idmap=container\" rootfs option: %w", err)
}
}
}
// Pass any cgroups rules into LXC.
if len(runConf.CGroups) > 0 {
for _, rule := range runConf.CGroups {
if strings.HasPrefix(rule.Key, "devices.") && (!d.isCurrentlyPrivileged() || d.state.OS.RunningInUserNS) {
continue
}
err = lxcSetConfigItem(cc, fmt.Sprintf("lxc.cgroup2.%s", rule.Key), rule.Value)
if err != nil {
return "", nil, fmt.Errorf("Failed to setup device cgroup %q: %w", dev.Name(), err)
}
}
}
// Pass any mounts into LXC.
if len(runConf.Mounts) > 0 {
escapePathFstab := func(mountPath string) string {
r := strings.NewReplacer(
" ", "\\040",
"\t", "\\011",
"\n", "\\012",
"\\", "\\\\",
)
return r.Replace(mountPath)
}
for _, mount := range runConf.Mounts {
if slices.Contains(mount.Opts, "propagation") && !liblxc.RuntimeLiblxcVersionAtLeast(liblxc.Version(), 3, 0, 0) {
return "", nil, fmt.Errorf("Failed to setup device mount %q: %w", dev.Name(), errors.New("liblxc 3.0 is required for mount propagation configuration"))
}
mntOptions := strings.Join(mount.Opts, ",")
if !d.IsPrivileged() && mount.OwnerShift == deviceConfig.MountOwnerShiftDynamic {
switch d.IdmappedStorage(mount.DevPath, mount.FSType) {
case idmap.StorageTypeIdmapped:
mntOptions = strings.Join([]string{mntOptions, "idmap=container"}, ",")
case idmap.StorageTypeNone:
return "", nil, fmt.Errorf("Failed to setup device mount %q: %w", dev.Name(), errors.New("idmapping abilities are required but aren't supported on system"))
}
}
mntVal := fmt.Sprintf("%s %s %s %s %d %d", escapePathFstab(mount.DevPath), escapePathFstab(mount.TargetPath), mount.FSType, mntOptions, mount.Freq, mount.PassNo)
err = lxcSetConfigItem(cc, "lxc.mount.entry", mntVal)
if err != nil {
return "", nil, fmt.Errorf("Failed to setup device mount %q: %w", dev.Name(), err)
}
}
}
// Pass any network setup config into LXC.
if len(runConf.NetworkInterface) > 0 {
// Increment nicID so that LXC network index is unique per device.
nicID++
networkKeyPrefix := "lxc.net"
if !liblxc.RuntimeLiblxcVersionAtLeast(liblxc.Version(), 2, 1, 0) {
networkKeyPrefix = "lxc.network"
}
for _, nicItem := range runConf.NetworkInterface {
// The connected state is not a LXC configuration key; we defer its handling to a post hook.
if nicItem.Key == "connected" {
runConf.PostHooks = append(runConf.PostHooks, func() error {
return d.setNICLink(dev.Name(), util.IsTrueOrEmpty(nicItem.Value), true)
})
continue
}
err = lxcSetConfigItem(cc, fmt.Sprintf("%s.%d.%s", networkKeyPrefix, nicID, nicItem.Key), nicItem.Value)
if err != nil {
return "", nil, fmt.Errorf("Failed to setup device network interface %q: %w", dev.Name(), err)
}
}
}
// Add any post start hooks.
if len(runConf.PostHooks) > 0 {
postStartHooks = append(postStartHooks, runConf.PostHooks...)
}
// Build list of NVIDIA GPUs (used for MIG).
if len(runConf.GPUDevice) > 0 {
for _, entry := range runConf.GPUDevice {
if entry.Key == device.GPUNvidiaDeviceKey {
nvidiaDevices = append(nvidiaDevices, entry.Value)
}
}
}
}
// Label rootfs if SELinux context is set and labels are missing.
err = d.selinuxLabelFiles(contextIsNew)
if err != nil {
return "", nil, err
}
// Initialize the credentials directory.
err = d.setupCredentials(false)
if err != nil {
return "", nil, err
}
// Override NVIDIA_VISIBLE_DEVICES if we have devices that need it.
if len(nvidiaDevices) > 0 {
err = lxcSetConfigItem(cc, "lxc.environment", fmt.Sprintf("\"NVIDIA_VISIBLE_DEVICES=%s\"", strings.Join(nvidiaDevices, ",")))
if err != nil {
return "", nil, fmt.Errorf("Unable to set NVIDIA_VISIBLE_DEVICES in LXC environment: %w", err)
}
}
// Handle application containers.
if util.PathExists(filepath.Join(d.Path(), "config.json")) {
// Parse the OCI config.
data, err := os.ReadFile(filepath.Join(d.Path(), "config.json"))
if err != nil {
return "", nil, err
}
var config ociSpecs.Spec
err = json.Unmarshal([]byte(data), &config)
if err != nil {
return "", nil, err
}
// Mark the container as an OCI container if not already set.
if !util.IsTrue(d.expandedConfig["volatile.container.oci"]) {
volatileSet["volatile.container.oci"] = "true"
}
// Allow unprivileged users to use ping.
if !d.state.OS.RunningInUserNS {
maxGid := int64(4294967294)
if !d.IsPrivileged() {
maxGid = 0
idMap, err := d.CurrentIdmap()
if err != nil {
return "", nil, err
}
for _, entry := range idMap.Entries {
if entry.NSID+entry.MapRange-1 > maxGid {
maxGid = entry.NSID + entry.MapRange - 1
}
}
}
err = lxcSetConfigItem(cc, "lxc.sysctl.net.ipv4.ping_group_range", fmt.Sprintf("0 %d", maxGid))
if err != nil {
return "", nil, err
}
}
// Allow unprivileged users to use low ports.
err = lxcSetConfigItem(cc, "lxc.sysctl.net.ipv4.ip_unprivileged_port_start", "0")
if err != nil {
return "", nil, err
}
// Configure the entry point.
entrypoint := config.Process.Args
if d.expandedConfig["oci.entrypoint"] != "" {
entrypoint, err = shellquote.Split(d.expandedConfig["oci.entrypoint"])
if err != nil {
return "", nil, err
}
}
// Compute the entrypoint string using LXC's own quoting rules.
initCmd, err := lxcEncodeCmd(entrypoint)
if err != nil {
return "", nil, err
}
if len(entrypoint) > 0 && slices.Contains([]string{"/init", "/sbin/init", "/s6-init", "/usr/bin/init"}, entrypoint[0]) {
// For regular init systems, call them directly as PID1.
err = lxcSetConfigItem(cc, "lxc.init.cmd", initCmd)
if err != nil {
return "", nil, err
}
} else {
// For anything else, run them under our own PID1.
err = lxcSetConfigItem(cc, "lxc.execute.cmd", initCmd)
if err != nil {
return "", nil, err
}
}
// Configure the cwd.
if d.expandedConfig["oci.cwd"] != "" {
err = lxcSetConfigItem(cc, "lxc.init.cwd", d.expandedConfig["oci.cwd"])
if err != nil {
return "", nil, err
}
} else {
err = lxcSetConfigItem(cc, "lxc.init.cwd", config.Process.Cwd)
if err != nil {
return "", nil, err
}
}
// Configure the UID
if d.expandedConfig["oci.uid"] != "" {
err = lxcSetConfigItem(cc, "lxc.init.uid", d.expandedConfig["oci.uid"])
if err != nil {
return "", nil, err
}
} else {
err = lxcSetConfigItem(cc, "lxc.init.uid", fmt.Sprintf("%d", config.Process.User.UID))
if err != nil {
return "", nil, err
}
}
// Configure the GID
if d.expandedConfig["oci.gid"] != "" {
err = lxcSetConfigItem(cc, "lxc.init.gid", d.expandedConfig["oci.gid"])
if err != nil {
return "", nil, err
}
} else {
err = lxcSetConfigItem(cc, "lxc.init.gid", fmt.Sprintf("%d", config.Process.User.GID))
if err != nil {
return "", nil, err
}
}
// Get all mounts so far.
lxcMounts := []string{"/dev", "/proc", "/sys", "/sys/fs/cgroup"}
for _, mount := range cc.ConfigItem("lxc.mount.entry") {
fields := strings.Split(mount, " ")
if len(fields) < 2 || fields[1][0] == '/' {
continue
}
lxcMounts = append(lxcMounts, filepath.Clean(fmt.Sprintf("/%s", fields[1])))
}
// Configure mounts.
for _, mount := range config.Mounts {
// We only support simple tmpfs at this stage.
if len(mount.UIDMappings) > 0 || len(mount.GIDMappings) > 0 || mount.Type != "tmpfs" {
continue
}
// Skip all our own mounts.
if slices.Contains(lxcMounts, filepath.Clean(mount.Destination)) {
continue
}
err := lxcSetConfigItem(cc, "lxc.mount.entry", fmt.Sprintf("%s %s %s %s 0 0", mount.Source, strings.TrimLeft(mount.Destination, "/"), mount.Type, strings.Join(append(mount.Options, "create=dir"), ",")))
if err != nil {
return "", nil, err
}
lxcMounts = append(lxcMounts, mount.Destination)
}
// Mount /run as a tmpfs if it exists and isn't already mounted.
if !slices.Contains(lxcMounts, "/run") {
err := lxcSetConfigItem(cc, "lxc.mount.entry", "none run tmpfs none,mode=755,optional")
if err != nil {
return "", nil, err
}
}
// Configure network handling.
// Confine all writes to the instance directory to avoid following image-planted symlinks.
instRoot, err := os.OpenRoot(d.Path())
if err != nil {
return "", nil, err
}
defer logger.WarnOnError(instRoot.Close, "Failed to close instance root")
err = instRoot.Mkdir("network", 0o711)
if err != nil && !errors.Is(err, fs.ErrExist) {
return "", nil, err
}
err = os.MkdirAll(filepath.Join(d.RootfsPath(), "etc"), 0o755)
if err != nil && !os.IsExist(err) {
return "", nil, err
}
err = instRoot.WriteFile("network/hosts", fmt.Appendf(nil, `127.0.0.1 localhost
127.0.1.1 %s
::1 localhost ip6-localhost ip6-loopback
fe00::0 ip6-localnet
ff00::0 ip6-mcastprefix
ff02::1 ip6-allnodes
ff02::2 ip6-allrouters
`, d.name), 0o644)
if err != nil {
return "", nil, err
}
err = lxcSetConfigItem(cc, "lxc.mount.entry", fmt.Sprintf("%s etc/hosts none bind,create=file", filepath.Join(d.Path(), "network", "hosts")))
if err != nil {
return "", nil, err
}
err = instRoot.WriteFile("network/hostname", fmt.Appendf(nil, "%s\n", d.name), 0o644)
if err != nil {
return "", nil, err
}
err = lxcSetConfigItem(cc, "lxc.mount.entry", fmt.Sprintf("%s etc/hostname none bind,create=file", filepath.Join(d.Path(), "network", "hostname")))
if err != nil {
return "", nil, err
}
// Generate the initial resolv.conf from the DNS settings (extended later over DHCP).
var resolvConf strings.Builder
for _, ns := range util.SplitNTrimSpace(d.expandedConfig["oci.dns.nameservers"], ",", -1, true) {
fmt.Fprintf(&resolvConf, "nameserver %s\n", ns)
}
if d.expandedConfig["oci.dns.search"] != "" {
fmt.Fprintf(&resolvConf, "search %s\n", strings.Join(util.SplitNTrimSpace(d.expandedConfig["oci.dns.search"], ",", -1, true), " "))
}
if d.expandedConfig["oci.dns.domain"] != "" {
fmt.Fprintf(&resolvConf, "domain %s\n", d.expandedConfig["oci.dns.domain"])
}
err = instRoot.WriteFile("network/resolv.conf", []byte(resolvConf.String()), 0o644)
if err != nil {
return "", nil, err
}
err = lxcSetConfigItem(cc, "lxc.mount.entry", fmt.Sprintf("%s etc/resolv.conf none bind,create=file", filepath.Join(d.Path(), "network", "resolv.conf")))
if err != nil {
return "", nil, err
}
// Record the expected network configuration for each interface.
ifaces := map[string]OCINetworkInterface{}
for _, dev := range d.expandedDevices.Sorted() {
if dev.Config["type"] != "nic" || dev.Config["name"] == "" {
continue
}
ifaces[dev.Config["name"]] = OCINetworkInterface{
DHCP4: dev.Config["ipv4.address"] != "none" && !strings.Contains(dev.Config["ipv4.address"], "/"),
DHCP6: dev.Config["ipv6.address"] != "none" && !strings.Contains(dev.Config["ipv6.address"], "/"),
Route4: dev.Config["ipv4.gateway"] != "none",
Route6: dev.Config["ipv6.gateway"] != "none",
}
}
ifacesData, err := json.Marshal(ifaces)
if err != nil {
return "", nil, err
}
err = instRoot.WriteFile("network/interfaces.json", ifacesData, 0o644)
if err != nil {
return "", nil, err
}
forknetDhcpLogfilePath := filepath.Join(d.LogPath(), "forknet-dhcp.log")
forknetDhcpLogfile, err := os.Create(forknetDhcpLogfilePath)
if err != nil {
return "", nil, err
}
err = forknetDhcpLogfile.Close()
if err != nil {
return "", nil, err
}
err = lxcSetConfigItem(cc, "lxc.hook.start-host", fmt.Sprintf(
"/proc/%d/exe forknet dhcp %s %s",
os.Getpid(),
filepath.Join(d.Path(), "network"),
forknetDhcpLogfilePath,
))
if err != nil {
return "", nil, err
}
} else {
// OCI specific configuration keys aren't valid on regular containers.
for key, value := range d.expandedConfig {
if value != "" && strings.HasPrefix(key, "oci.") {
return "", nil, fmt.Errorf("%q is only supported on OCI containers", key)
}
}
// Clear OCI config key if present.
if d.expandedConfig["volatile.container.oci"] != "" {
volatileSet["volatile.container.oci"] = ""
}
}
// Check if we should start a dedicated LXCFS.
if d.state.GlobalConfig.InstancesLXCFSPerInstance() {
if !util.PathExists(filepath.Join(d.RunPath(), "lxcfs", "proc")) {
// Make sure all the paths exist.
err := os.Mkdir(filepath.Join(d.DevicesPath(), "lxcfs"), 0o711)
if err != nil && !os.IsExist(err) {
return "", nil, err
}
err = os.Mkdir(filepath.Join(d.RunPath(), "lxcfs"), 0o700)
if err != nil && !os.IsExist(err) {
return "", nil, err
}
// Prepare a new LXCFS instance.
args := []string{
"-f",
"-p", filepath.Join(d.RunPath(), "lxcfs.pid"),
"--runtime-dir", filepath.Join(d.RunPath(), "lxcfs"),
}
if os.Getenv("LXCFS_OPTS") != "" {
userArgs, err := shellquote.Split(os.Getenv("LXCFS_OPTS"))
if err != nil {
return "", nil, err
}
args = append(args, userArgs...)
}
args = append(args, filepath.Join(d.DevicesPath(), "lxcfs"))
lxcfs, err := subprocess.NewProcess("lxcfs", args, "", "")
if err != nil {
return "", nil, err
}
// Start LXCFS.
err = lxcfs.Start(context.TODO())
if err != nil {
return "", nil, err
}
// Write down our process tracking.
err = lxcfs.Save(filepath.Join(d.RunPath(), "lxcfs.yaml"))
if err != nil {
return "", nil, err
}
}
// Over-mount the system LXCFS (if found).
for _, entry := range []string{"/var/lib/lxcfs", "/var/lib/incus-lxcfs"} {
if !util.PathExists(entry) {
continue
}
err = lxcSetConfigItem(cc, "lxc.hook.pre-mount", fmt.Sprintf("mount -o bind %s %s/", filepath.Join(d.DevicesPath(), "lxcfs"), entry))
if err != nil {
return "", nil, err
}
}
}
// Setup BPF token delegation if enabled
bpfConfig := d.bpfTokenConfig()
if bpfConfig.enable {
err = lxcSetConfigItem(cc, "lxc.hook.start-host", shellquote.Join(
fmt.Sprintf("/proc/%d/exe", os.Getpid()),
"forkbpf",
bpfConfig.mountPath,
bpfConfig.cmdTypes,
bpfConfig.mapTypes,
bpfConfig.progTypes,
bpfConfig.attachTypes,
))
if err != nil {
return "", nil, err
}
}
// Load the LXC raw config.
err = d.loadRawLXCConfig(cc)
if err != nil {
return "", nil, err
}
// Generate the LXC config
configPath := filepath.Join(d.RunPath(), "lxc.conf")
err = cc.SaveConfigFile(configPath)
if err != nil {
_ = os.Remove(configPath)
return "", nil, err
}
// Set ownership to match container root
currentIdmapset, err := d.CurrentIdmap()
if err != nil {
return "", nil, err
}
uid := int64(0)
if currentIdmapset != nil {
uid, _ = currentIdmapset.ShiftFromNS(0, 0)
}
err = os.Chown(d.Path(), int(uid), 0)
if err != nil {
return "", nil, err
}
// We only need traversal by root in the container
err = os.Chmod(d.Path(), 0o100)
if err != nil {
return "", nil, err
}
// If starting stateless, wipe state
if !d.IsStateful() && util.PathExists(d.StatePath()) {
_ = os.RemoveAll(d.StatePath())
}
// Snapshot if needed.
snapName, expiry, err := d.getStartupSnapNameAndExpiry(d)
if err != nil {
return "", nil, fmt.Errorf("Failed getting startup snapshot info: %w", err)
}
if snapName != "" && expiry != nil {
err := d.snapshot(snapName, *expiry, false)
if err != nil {
return "", nil, fmt.Errorf("Failed taking startup snapshot: %w", err)
}
}
// Apply any volatile changes that need to be made.
err = d.VolatileSet(volatileSet)
if err != nil {
return "", nil, fmt.Errorf("Failed setting volatile keys: %w", err)
}
// Update the backup.yaml file just before starting the instance process, but after all devices have been
// setup, so that the backup file contains the volatile keys used for this instance start, so that they
// can be used for instance cleanup.
err = d.UpdateBackupFile()
if err != nil {
return "", nil, err
}
reverter.Success()
return configPath, postStartHooks, nil
}
type bpfTokenConfig struct {
enable bool
mountPath string
cmdTypes string
mapTypes string
progTypes string
attachTypes string
}
func (d *lxc) bpfTokenConfig() bpfTokenConfig {
if util.IsTrue(d.expandedConfig["security.privileged"]) {
return bpfTokenConfig{}
}
cfg := bpfTokenConfig{
mountPath: d.expandedConfig["security.bpffs.path"],
cmdTypes: d.expandedConfig["security.bpffs.delegate_cmds"],
mapTypes: d.expandedConfig["security.bpffs.delegate_maps"],
progTypes: d.expandedConfig["security.bpffs.delegate_progs"],
attachTypes: d.expandedConfig["security.bpffs.delegate_attachs"],
}
if cfg.cmdTypes != "" || cfg.mapTypes != "" || cfg.progTypes != "" || cfg.attachTypes != "" {
cfg.enable = true
}
if cfg.mountPath == "" {
cfg.mountPath = "/sys/fs/bpf"
}
return cfg
}
// detachInterfaceRename enters the container's network namespace and moves the named interface
// in ifName back to the network namespace of the running process as the name specified in hostName.
func (d *lxc) detachInterfaceRename(netns string, ifName string, hostName string) error {
daemonPID := os.Getpid()
// Run forknet detach
_, err := subprocess.RunCommand(
d.state.OS.ExecPath,
"forknet",
"detach",
"--",
netns,
fmt.Sprintf("%d", daemonPID),
ifName,
hostName,
)
// Process forknet detach response
if err != nil {
return err
}
return nil
}
// Start starts the instance.
func (d *lxc) Start(stateful bool) error {
// Check that migration.stateful is set for stateful actions.
if stateful && !d.CanLiveMigrate() {
return errors.New("Stateful start requires that the instance migration.stateful be set to true")
}
d.logger.Debug("Start started", logger.Ctx{"stateful": stateful})
defer d.logger.Debug("Start finished", logger.Ctx{"stateful": stateful})
// Check that we are startable before creating an operation lock.
// Must happen before creating operation Start lock to avoid the status check returning Stopped due to the
// existence of a Start operation lock.
err := d.validateStartup(stateful, d.statusCode())
if err != nil {
return err
}
// Setup a new operation.
op, err := operationlock.CreateWaitGet(d.Project().Name, d.Name(), d.op, operationlock.ActionStart, []operationlock.Action{operationlock.ActionRestart, operationlock.ActionRestore}, false, false)
if err != nil {
if errors.Is(err, operationlock.ErrNonReusuableSucceeded) {
// An existing matching operation has now succeeded, return.
return nil
}
return fmt.Errorf("Failed to create instance start operation: %w", err)
}
defer op.Done(nil)
if !daemon.SharedMountsSetup {
err = errors.New("Daemon failed to setup shared mounts base. Does security.nesting need to be turned on?")
op.Done(err)
return err
}
ctxMap := logger.Ctx{
"action": op.Action(),
"created": d.creationDate,
"ephemeral": d.ephemeral,
"used": d.lastUsedDate,
"stateful": stateful,
}
if op.Action() == "start" {
d.logger.Info("Starting instance", ctxMap)
}
// If stateful, restore now.
if stateful && d.stateful {
d.logger.Info("Restoring stateful checkpoint")
criuMigrationArgs := instance.CriuMigrationArgs{
Cmd: liblxc.MIGRATE_RESTORE,
StateDir: d.StatePath(),
Function: "snapshot",
Stop: false,
ActionScript: false,
DumpDir: "",
PreDumpDir: "",
}
err = d.migrate(&criuMigrationArgs)
if err != nil && !d.IsRunning() {
op.Done(err)
return fmt.Errorf("Failed restoring stateful checkpoint: %w", err)
}
_ = os.RemoveAll(d.StatePath())
d.stateful = false
err = d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error {
return tx.UpdateInstanceStatefulFlag(ctx, d.id, false)
})
if err != nil {
op.Done(err)
return fmt.Errorf("Failed clearing instance stateful flag: %w", err)
}
if op.Action() == "start" {
d.logger.Info("Started instance", ctxMap)
d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceStarted.Event(d, nil))
}
return nil
} else if d.stateful {
/* stateless start required when we have state, let's delete it */
err := os.RemoveAll(d.StatePath())
if err != nil {
op.Done(err)
return err
}
d.stateful = false
err = d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error {
return tx.UpdateInstanceStatefulFlag(ctx, d.id, false)
})
if err != nil {
op.Done(err)
return fmt.Errorf("Failed clearing instance stateful flag: %w", err)
}
}
// Run the shared start code.
configPath, postStartHooks, err := d.startCommon()
if err != nil {
op.Done(err)
return err
}
name := project.Instance(d.Project().Name, d.name)
// Setup minimal environment for forklxc.
envDict := map[string]string{
"container": "lxc",
}
for k, v := range d.expandedConfig {
after, ok := strings.CutPrefix(k, "environment.")
if ok {
envDict[after] = v
}
}
for _, keepEnv := range []string{"LD_LIBRARY_PATH", "INCUS_DIR", "INCUS_SOCKET"} {
if os.Getenv(keepEnv) != "" {
envDict[keepEnv] = os.Getenv(keepEnv)
}
}
_, ok := envDict["PATH"]
if !ok {
envDict["PATH"] = os.Getenv("PATH")
}
env := make([]string, 0, len(envDict))
for k, v := range envDict {
env = append(env, fmt.Sprintf("%s=%s", k, v))
}
// Start the LXC container.
_, _, err = subprocess.RunCommandSplit(
context.TODO(),
env,
nil,
d.state.OS.ExecPath,
"forklxc",
name,
d.state.OS.LxcPath,
configPath,
d.LogPath(),
)
if err != nil && !d.IsRunning() {
// Attempt to extract the LXC errors
lxcLog := ""
logPath := filepath.Join(d.LogPath(), "lxc.log")
if util.PathExists(logPath) {
logContent, err := os.ReadFile(logPath)
if err == nil {
for _, line := range strings.Split(string(logContent), "\n") {
fields := strings.Fields(line)
if len(fields) < 4 {
continue
}
// We only care about errors
if fields[2] != "ERROR" {
continue
}
// Prepend the line break
if len(lxcLog) == 0 {
lxcLog += "\n"
}
lxcLog += fmt.Sprintf(" %s\n", strings.Join(fields[0:], " "))
}
}
}
d.logger.Error("Failed starting instance", ctxMap)
// Return the actual error
op.Done(err)
return err
}
// Run any post start hooks.
err = d.runHooks(postStartHooks)
if err != nil {
op.Done(err) // Must come before Stop() otherwise stop will not proceed.
// Attempt to stop container.
_ = d.Stop(false)
return err
}
// Apply OOM priority after container is started and hooks completed.
err = d.setOOMPriority(d.InitPID())
if err != nil {
d.logger.Warn("Failed to set OOM priority", logger.Ctx{
"err": err,
"instance": d.Name(),
"project": d.Project().Name,
})
}
if op.Action() == "start" {
d.logger.Info("Started instance", ctxMap)
d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceStarted.Event(d, nil))
}
return nil
}
// OnHook is the top-level hook handler.
func (d *lxc) OnHook(hookName string, args map[string]string) error {
switch hookName {
case instance.HookStart:
return d.onStart(args)
case instance.HookStopNS:
return d.onStopNS(args)
case instance.HookStop:
return d.onStop(args)
default:
return instance.ErrNotImplemented
}
}
// onStart implements the start hook.
func (d *lxc) onStart(_ map[string]string) error {
// Make sure we can't call go-lxc functions by mistake
d.fromHook = true
// Load the container AppArmor profile
err := apparmor.InstanceLoad(d.state.OS, d, nil)
if err != nil {
return err
}
// Template anything that needs templating
key := "volatile.apply_template"
if d.localConfig[key] != "" {
// Run any template that needs running
err = d.templateApplyNow(instance.TemplateTrigger(d.localConfig[key]))
if err != nil {
_ = apparmor.InstanceUnload(d.state.OS, d)
return err
}
err := d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error {
// Remove the volatile key from the DB
return tx.DeleteInstanceConfigKey(ctx, int64(d.id), key)
})
if err != nil {
_ = apparmor.InstanceUnload(d.state.OS, d)
return err
}
}
err = d.templateApplyNow("start")
if err != nil {
_ = apparmor.InstanceUnload(d.state.OS, d)
return err
}
// Trigger a rebalance
defer cgroup.TaskSchedulerTrigger("container", d.name, "started")
// Record last start state.
err = d.recordLastState()
if err != nil {
return err
}
return nil
}
// validateStartup checks any constraints that would prevent start up from succeeding under normal circumstances.
func (d *lxc) validateStartup(stateful bool, statusCode api.StatusCode) error {
err := d.common.validateStartup(stateful, statusCode)
if err != nil {
return err
}
// gendoc:generate(entity=image, group=requirements, key=requirements.nesting)
//
// ---
// type: bool
// shortdesc: If set to `true`, indicates that the image cannot work without nesting enabled.
//
// Ensure nesting is turned on for images that require nesting.
if util.IsTrue(d.localConfig["image.requirements.nesting"]) && util.IsFalseOrEmpty(d.expandedConfig["security.nesting"]) {
return errors.New("The image used by this instance requires nesting. Please set security.nesting=true on the instance")
}
return nil
}
// Stop functions.
func (d *lxc) Stop(stateful bool) error {
d.logger.Debug("Stop started", logger.Ctx{"stateful": stateful})
defer d.logger.Debug("Stop finished", logger.Ctx{"stateful": stateful})
// Check that migration.stateful is set for stateful actions.
if stateful && !d.CanLiveMigrate() {
return errors.New("Stateful stop requires the instance to have migration.stateful be set to true")
}
// Must be run prior to creating the operation lock.
if !d.IsRunning() {
return ErrInstanceIsStopped
}
// Setup a new operation
op, err := operationlock.CreateWaitGet(d.Project().Name, d.Name(), d.op, operationlock.ActionStop, []operationlock.Action{operationlock.ActionRestart, operationlock.ActionRestore, operationlock.ActionMigrate}, false, true)
if err != nil {
if errors.Is(err, operationlock.ErrNonReusuableSucceeded) {
// An existing matching operation has now succeeded, return.
return nil
}
return err
}
ctxMap := logger.Ctx{
"action": op.Action(),
"created": d.creationDate,
"ephemeral": d.ephemeral,
"used": d.lastUsedDate,
"stateful": stateful,
}
if op.Action() == "stop" {
d.logger.Info("Stopping instance", ctxMap)
}
// Forcefully stop any forkfile process if running.
d.stopForkfile(true)
// Release liblxc container once done.
defer func() {
d.release()
}()
// Load the go-lxc struct
var cc *liblxc.Container
if d.expandedConfig["raw.lxc"] != "" {
cc, err = d.initLXC(true)
if err != nil {
op.Done(err)
return err
}
err = d.loadRawLXCConfig(cc)
if err != nil {
op.Done(err)
return err
}
} else {
cc, err = d.initLXC(false)
if err != nil {
op.Done(err)
return err
}
}
// Handle stateful stop
if stateful {
// Cleanup any existing state
stateDir := d.StatePath()
_ = os.RemoveAll(stateDir)
err := os.MkdirAll(stateDir, 0o700)
if err != nil {
op.Done(err)
return err
}
criuMigrationArgs := instance.CriuMigrationArgs{
Cmd: liblxc.MIGRATE_DUMP,
StateDir: stateDir,
Function: "snapshot",
Stop: true,
ActionScript: false,
DumpDir: "",
PreDumpDir: "",
}
// Checkpoint
err = d.migrate(&criuMigrationArgs)
if err != nil {
op.Done(err)
return err
}
err = op.Wait(context.Background())
if err != nil && d.IsRunning() {
return err
}
d.stateful = true
err = d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error {
return tx.UpdateInstanceStatefulFlag(ctx, d.id, true)
})
if err != nil {
return fmt.Errorf("Failed updating instance stateful flag: %w", err)
}
d.logger.Info("Stopped instance", ctxMap)
d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceStopped.Event(d, nil))
return nil
} else if util.PathExists(d.StatePath()) {
_ = os.RemoveAll(d.StatePath())
}
// Load cgroup abstraction
cg, err := d.cgroup(cc, true)
if err != nil {
op.Done(err)
return err
}
// Fork-bomb mitigation, prevent forking from this point on
if cgroup.Supports(cgroup.Pids) {
// Attempt to disable forking new processes
_ = cg.SetMaxProcesses(0)
} else {
// Attempt to freeze the container
freezer := make(chan bool, 1)
go func() {
_ = d.Freeze()
freezer <- true
}()
select {
case <-freezer:
case <-time.After(time.Second * 5):
_ = d.Unfreeze()
}
}
err = cc.Stop()
if err != nil {
op.Done(err)
return err
}
// Wait for operation lock to be Done. This is normally completed by onStop which picks up the same
// operation lock and then marks it as Done after the instance stops and the devices have been cleaned up.
// However if the operation has failed for another reason we will collect the error here.
err = op.Wait(context.Background())
status := d.statusCode()
if status != api.Stopped {
errPrefix := fmt.Errorf("Failed stopping instance, status is %q", status)
if err != nil {
return fmt.Errorf("%s: %w", errPrefix.Error(), err)
}
return errPrefix
} else if op.Action() == "stop" {
// If instance stopped, send lifecycle event (even if there has been an error cleaning up).
d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceStopped.Event(d, nil))
}
// Now handle errors from stop sequence and return to caller if wasn't completed cleanly.
if err != nil {
return err
}
return nil
}
// Shutdown stops the instance.
func (d *lxc) Shutdown(timeout time.Duration) error {
d.logger.Debug("Shutdown started", logger.Ctx{"timeout": timeout})
defer d.logger.Debug("Shutdown finished", logger.Ctx{"timeout": timeout})
// Must be run prior to creating the operation lock.
statusCode := d.statusCode()
if !d.isRunningStatusCode(statusCode) {
if statusCode == api.Error {
return fmt.Errorf("The instance cannot be cleanly shutdown as in %s status", statusCode)
}
return ErrInstanceIsStopped
}
// Setup a new operation
op, err := operationlock.CreateWaitGet(d.Project().Name, d.Name(), d.op, operationlock.ActionStop, []operationlock.Action{operationlock.ActionRestart}, true, true)
if err != nil {
if errors.Is(err, operationlock.ErrNonReusuableSucceeded) {
// An existing matching operation has now succeeded, return.
return nil
}
return err
}
// If frozen, resume so the signal can be handled.
if d.IsFrozen() {
err := d.Unfreeze()
if err != nil {
return err
}
// Wait 3s for init to be running enough to get the next signal handle.
time.Sleep(3 * time.Second)
}
ctxMap := logger.Ctx{
"action": "shutdown",
"created": d.creationDate,
"ephemeral": d.ephemeral,
"used": d.lastUsedDate,
"timeout": timeout,
}
if op.Action() == "stop" {
d.logger.Info("Shutting down instance", ctxMap)
}
// Release liblxc container once done.
defer func() {
d.release()
}()
// Load the go-lxc struct
var cc *liblxc.Container
if d.expandedConfig["raw.lxc"] != "" {
cc, err = d.initLXC(true)
if err != nil {
op.Done(err)
return err
}
err = d.loadRawLXCConfig(cc)
if err != nil {
op.Done(err)
return err
}
} else {
cc, err = d.initLXC(false)
if err != nil {
op.Done(err)
return err
}
}
// Request shutdown, but don't wait for container to stop. If call fails then cancel operation with error,
// otherwise expect the onStop() hook to cancel operation when done (when the container has stopped).
err = cc.Shutdown(0)
if err != nil {
op.Done(err)
}
d.logger.Debug("Shutdown request sent to instance")
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
// Wait for operation lock to be Done or context to timeout. The operation lock is normally completed by
// onStop which picks up the same lock and then marks it as Done after the instance stops and the devices
// have been cleaned up. However if the operation has failed for another reason we collect the error here.
err = op.Wait(ctx)
status := d.statusCode()
if status != api.Stopped {
errPrefix := fmt.Errorf("Failed shutting down instance, status is %q", status)
if err != nil {
return fmt.Errorf("%s: %w", errPrefix.Error(), err)
}
return errPrefix
} else if op.Action() == "stop" {
// If instance stopped, send lifecycle event (even if there has been an error cleaning up).
d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceShutdown.Event(d, nil))
}
// Now handle errors from shutdown sequence and return to caller if wasn't completed cleanly.
if err != nil {
return err
}
return nil
}
// Restart restart the instance.
func (d *lxc) Restart(timeout time.Duration) error {
return d.restartCommon(d, timeout)
}
// Rebuild rebuilds the instance using the supplied image fingerprint as source.
func (d *lxc) Rebuild(img *api.Image, op *operations.Operation) error {
return d.rebuildCommon(d, img, op)
}
// stopDHCPClient kills the forknet dhcp process if any and waits for it to
// exit so the container's cgroup can be fully cleaned up.
func (d *lxc) stopDHCPClient() {
pidPath := filepath.Join(d.Path(), "network", "dhcp.pid")
dhcpPIDStr, err := os.ReadFile(pidPath)
if err != nil {
return
}
dhcpPID, err := strconv.Atoi(strings.TrimSpace(string(dhcpPIDStr)))
if err != nil {
return
}
pidFd, err := linux.PidFdOpen(dhcpPID, 0)
if err != nil {
return
}
defer func() { _ = pidFd.Close() }()
// Guard against PID reuse.
cmdline, err := os.ReadFile(fmt.Sprintf("/proc/%d/cmdline", dhcpPID))
if err != nil || (!strings.HasPrefix(string(cmdline), "[incus dhcp]") && !strings.Contains(string(cmdline), "forknet\x00dhcp")) {
return
}
err = linux.PidfdSendSignal(int(pidFd.Fd()), int(unix.SIGTERM), 0)
if err != nil {
return
}
// Wait for the process to exit.
fds := []unix.PollFd{{Fd: int32(pidFd.Fd()), Events: unix.POLLIN}}
_, _ = unix.Poll(fds, 5000)
_ = os.Remove(pidPath)
}
// onStopNS is triggered by LXC's stop hook once a container is shutdown but before the container's
// namespaces have been closed. The netns path of the stopped container is provided.
func (d *lxc) onStopNS(args map[string]string) error {
target := args["target"]
netns := args["netns"]
// Validate target.
if !slices.Contains([]string{"stop", "reboot"}, target) {
d.logger.Error("Container sent invalid target to OnStopNS", logger.Ctx{"target": target})
return fmt.Errorf("Invalid stop target %q", target)
}
// Create/pick up operation, but don't complete it as we leave operation running for the onStop hook below.
_, err := d.onStopOperationSetup(target)
if err != nil {
return err
}
// Stop the DHCP client if any.
d.stopDHCPClient()
// Clean up devices.
d.cleanupDevices(false, netns)
return nil
}
// onStop is triggered by LXC's post-stop hook once a container is shutdown and after the
// container's namespaces have been closed.
func (d *lxc) onStop(args map[string]string) error {
target := args["target"]
// Validate target
if !slices.Contains([]string{"stop", "reboot"}, target) {
d.logger.Error("Container sent invalid target to OnStop", logger.Ctx{"target": target})
return fmt.Errorf("Invalid stop target: %s", target)
}
// Create/pick up operation.
op, err := d.onStopOperationSetup(target)
if err != nil {
return err
}
// Make sure we can't call go-lxc functions by mistake
d.fromHook = true
// Record power state.
err = d.VolatileSet(map[string]string{
"volatile.last_state.power": instance.PowerStateStopped,
"volatile.last_state.ready": "false",
})
if err != nil {
// Don't return an error here as we still want to cleanup the instance even if DB not available.
d.logger.Error("Failed recording last power state", logger.Ctx{"err": err})
}
go func(d *lxc, target string, op *operationlock.InstanceOperation) {
d.fromHook = false
err = nil
// Set operation if missing.
if d.op == nil {
d.op = op.GetOperation()
}
// Unlock on return
defer op.Done(nil)
d.logger.Debug("Instance stopped, cleaning up")
// Wait for any file operations to complete.
// This is to required so we can actually unmount the container.
d.stopForkfile(false)
// Clean up devices.
d.cleanupDevices(false, "")
// Stop the DHCP client if it's somehow still around.
d.stopDHCPClient()
// Remove directory ownership (to avoid issue if uidmap is reused)
err := os.Chown(d.Path(), 0, 0)
if err != nil {
op.Done(fmt.Errorf("Failed clearing ownership: %w", err))
return
}
err = os.Chmod(d.Path(), 0o100)
if err != nil {
op.Done(fmt.Errorf("Failed clearing permissions: %w", err))
return
}
// Stop the storage for this container
err = d.unmount()
if err != nil && !errors.Is(err, storageDrivers.ErrInUse) {
err = fmt.Errorf("Failed unmounting instance: %w", err)
op.Done(err)
return
}
// Unload the apparmor profile
err = apparmor.InstanceUnload(d.state.OS, d)
if err != nil {
op.Done(fmt.Errorf("Failed to destroy apparmor namespace: %w", err))
return
}
// Clean all the unix devices
err = d.removeUnixDevices()
if err != nil {
op.Done(fmt.Errorf("Failed to remove unix devices: %w", err))
return
}
// Clean all the disk devices
err = d.removeDiskDevices()
if err != nil {
op.Done(fmt.Errorf("Failed to remove disk devices: %w", err))
return
}
// Stop dedicated LXCFS.
if util.PathExists(filepath.Join(d.DevicesPath(), "lxcfs", "proc")) && util.PathExists(filepath.Join(d.RunPath(), "lxcfs.yaml")) {
// Import the running LXCFS.
lxcfs, err := subprocess.ImportProcess(filepath.Join(d.RunPath(), "lxcfs.yaml"))
if err != nil && !os.IsExist(err) {
op.Done(fmt.Errorf("Failed to stop LXCFS: %w", err))
return
}
// Stop LXCFS.
err = lxcfs.Stop()
if err != nil && !errors.Is(err, subprocess.ErrNotRunning) {
op.Done(fmt.Errorf("Failed to stop LXCFS: %w", err))
return
}
_ = unix.Unmount(filepath.Join(d.DevicesPath(), "lxcfs"), unix.MNT_DETACH)
}
// Determine if instance should be auto-restarted.
var autoRestart bool
if target != "reboot" && op.GetInstanceInitiated() && d.shouldAutoRestart() {
autoRestart = true
// Mark current shutdown as complete.
op.Done(nil)
// Create a new restart operation.
op, err = operationlock.CreateWaitGet(d.Project().Name, d.Name(), d.op, operationlock.ActionRestart, nil, true, false)
if err == nil {
defer op.Done(nil)
} else {
d.logger.Error("Failed to setup new restart operation", logger.Ctx{"err": err})
}
}
// Log and emit lifecycle if not user triggered
if target != "reboot" && !autoRestart && op.GetInstanceInitiated() {
ctxMap := logger.Ctx{
"action": target,
"created": d.creationDate,
"ephemeral": d.ephemeral,
"used": d.lastUsedDate,
"stateful": false,
}
d.logger.Info("Shut down instance", ctxMap)
d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceShutdown.Event(d, nil))
}
// Reboot the container
if target == "reboot" || autoRestart {
// Start the container again
err = d.Start(false)
if err != nil {
op.Done(fmt.Errorf("Failed restarting instance: %w", err))
return
}
d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceRestarted.Event(d, nil))
return
}
// Trigger a rebalance
defer cgroup.TaskSchedulerTrigger("container", d.name, "stopped")
// Destroy ephemeral containers
if d.ephemeral {
err = d.delete(true, true)
if err != nil {
op.Done(fmt.Errorf("Failed deleting ephemeral instance: %w", err))
return
}
}
}(d, target, op)
return nil
}
// cleanupDevices performs any needed device cleanup steps when container is stopped.
// Accepts a stopHookNetnsPath argument which is required when run from the onStopNS hook before the
// container's network namespace is unmounted (which is required for NIC device cleanup).
func (d *lxc) cleanupDevices(instanceRunning bool, stopHookNetnsPath string) {
for _, entry := range d.expandedDevices.Reversed() {
// Only stop NIC devices when run from the onStopNS hook, and stop all other devices when run from
// the onStop hook. This way disk devices are stopped after the instance has been fully stopped.
if (stopHookNetnsPath != "" && entry.Config["type"] != "nic") || (stopHookNetnsPath == "" && entry.Config["type"] == "nic") {
continue
}
dev, err := d.deviceLoad(d, entry.Name, entry.Config, false)
if err != nil {
if errors.Is(err, device.ErrUnsupportedDevType) {
continue // Skip unsupported device (allows for mixed instance type profiles).
}
// Just log an error, but still allow the device to be stopped if usable device returned.
d.logger.Error("Failed stop validation for device", logger.Ctx{"device": entry.Name, "err": err})
}
// If a usable device was returned from deviceLoad try to stop anyway, even if validation fails.
// This allows for the scenario where a new version has additional validation restrictions
// than older versions and we still need to allow previously valid devices to be stopped even if
// they are no longer considered valid.
if dev != nil {
err = d.deviceStop(dev, instanceRunning, stopHookNetnsPath)
if err != nil {
d.logger.Error("Failed to stop device", logger.Ctx{"device": dev.Name(), "err": err})
}
}
}
}
// cleanupFailedMigrationRestore removes devices prepared by startCommon when CRIU restore fails before the stop hooks can run.
func (d *lxc) cleanupFailedMigrationRestore() {
d.cleanupDevices(false, "")
err := d.removeUnixDevices()
if err != nil {
d.logger.Error("Failed to remove Unix devices after migration restore failure", logger.Ctx{"err": err})
}
err = d.removeDiskDevices()
if err != nil {
d.logger.Error("Failed to remove disk devices after migration restore failure", logger.Ctx{"err": err})
}
}
// Freeze functions.
func (d *lxc) Freeze() error {
ctxMap := logger.Ctx{
"created": d.creationDate,
"ephemeral": d.ephemeral,
"used": d.lastUsedDate,
}
// Check that we're running
if !d.IsRunning() {
return errors.New("The instance isn't running")
}
// Load the go-lxc struct
cc, err := d.initLXC(false)
if err != nil {
ctxMap["err"] = err
d.logger.Error("Failed freezing container", ctxMap)
return err
}
// Check that we're not already frozen
if d.IsFrozen() {
return errors.New("The container is already frozen")
}
d.logger.Info("Freezing container", ctxMap)
err = cc.Freeze()
if err != nil {
ctxMap["err"] = err
d.logger.Error("Failed freezing container", ctxMap)
return err
}
d.logger.Info("Froze container", ctxMap)
d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstancePaused.Event(d, nil))
return err
}
// Unfreeze unfreezes the instance.
func (d *lxc) Unfreeze() error {
ctxMap := logger.Ctx{
"created": d.creationDate,
"ephemeral": d.ephemeral,
"used": d.lastUsedDate,
}
// Check that we're running
if !d.IsRunning() {
return errors.New("The container isn't running")
}
// Load the go-lxc struct
cc, err := d.initLXC(false)
if err != nil {
d.logger.Error("Failed unfreezing container", ctxMap)
return err
}
// Check that we're frozen
if !d.IsFrozen() {
return errors.New("The container is already running")
}
d.logger.Info("Unfreezing container", ctxMap)
err = cc.Unfreeze()
if err != nil {
d.logger.Error("Failed unfreezing container", ctxMap)
}
d.logger.Info("Unfroze container", ctxMap)
d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceResumed.Event(d, nil))
return err
}
// Get lxc container state, with 1 second timeout.
// If we don't get a reply, assume the lxc monitor is unresponsive.
func (d *lxc) getLxcState() (liblxc.State, error) {
if d.IsSnapshot() {
return liblxc.StateMap["STOPPED"], nil
}
// Load the go-lxc struct
cc, err := d.initLXC(false)
if err != nil {
return liblxc.StateMap["STOPPED"], err
}
monitor := make(chan liblxc.State, 1)
go func(c *liblxc.Container) {
monitor <- c.State()
}(cc)
select {
case lxcState := <-monitor:
return lxcState, nil
case <-time.After(5 * time.Second):
return liblxc.StateMap["FROZEN"], errors.New("Monitor is unresponsive")
}
}
// RenderWithUsage renders the API response including disk usage.
func (d *lxc) RenderWithUsage() (any, any, error) {
resp, etag, err := d.Render()
if err != nil {
return nil, nil, err
}
// Currently only snapshot data needs usage added.
snapResp, ok := resp.(*api.InstanceSnapshot)
if !ok {
return resp, etag, nil
}
pool, err := d.getStoragePool()
if err != nil {
return nil, nil, err
}
// It is important that the snapshot not be mounted here as mounting a snapshot can trigger a very
// expensive filesystem UUID regeneration, so we rely on the driver implementation to get the info
// we are requesting as cheaply as possible.
volumeState, err := pool.GetInstanceUsage(d)
if err != nil {
return resp, etag, nil
}
// A negative usage means the driver couldn't determine it, so leave the size unset.
if volumeState.Used >= 0 {
snapResp.Size = volumeState.Used
}
return snapResp, etag, nil
}
// Render renders the state of the instance.
func (d *lxc) Render() (any, any, error) {
// Ignore err as the arch string on error is correct (unknown)
architectureName, _ := osarch.ArchitectureName(d.architecture)
profileNames := make([]string, 0, len(d.profiles))
for _, profile := range d.profiles {
profileNames = append(profileNames, profile.Name)
}
if d.IsSnapshot() {
// Prepare the response.
snapState := api.InstanceSnapshot{
CreatedAt: d.creationDate,
Description: d.description,
ExpandedConfig: d.expandedConfig,
ExpandedDevices: d.expandedDevices.CloneNative(),
LastUsedAt: d.lastUsedDate,
Name: strings.SplitN(d.name, "/", 2)[1],
Stateful: d.stateful,
Size: -1, // Default to uninitialized/error state (0 means no CoW usage).
}
snapState.Architecture = architectureName
snapState.Config = d.localConfig
snapState.Devices = d.localDevices.CloneNative()
snapState.Ephemeral = d.ephemeral
snapState.Profiles = profileNames
snapState.ExpiresAt = d.expiryDate
return &snapState, d.ETag(), nil
}
// Prepare the response.
statusCode := d.statusCode()
instState := api.Instance{
ExpandedConfig: d.expandedConfig,
ExpandedDevices: d.expandedDevices.CloneNative(),
Name: d.name,
Status: statusCode.String(),
StatusCode: statusCode,
Location: d.node,
Type: d.Type().String(),
}
instState.Description = d.description
instState.Architecture = architectureName
instState.Config = d.localConfig
instState.CreatedAt = d.creationDate
instState.Devices = d.localDevices.CloneNative()
instState.Ephemeral = d.ephemeral
instState.LastUsedAt = d.lastUsedDate
instState.Profiles = profileNames
instState.Stateful = d.stateful
instState.Project = d.project.Name
return &instState, d.ETag(), nil
}
// RenderFull renders the full state of the instance.
func (d *lxc) RenderFull(hostInterfaces []net.Interface) (*api.InstanceFull, any, error) {
if d.IsSnapshot() {
return nil, nil, errors.New("RenderFull only works with containers")
}
// Get the Container struct
base, etag, err := d.Render()
if err != nil {
return nil, nil, err
}
// Convert to ContainerFull
ct := api.InstanceFull{Instance: *base.(*api.Instance)}
// Add the ContainerState
ct.State, err = d.renderState(ct.StatusCode, hostInterfaces)
if err != nil {
return nil, nil, err
}
// Add the ContainerSnapshots
snaps, err := d.Snapshots()
if err != nil {
return nil, nil, err
}
for _, snap := range snaps {
render, _, err := snap.Render()
if err != nil {
return nil, nil, err
}
if ct.Snapshots == nil {
ct.Snapshots = []api.InstanceSnapshot{}
}
ct.Snapshots = append(ct.Snapshots, *render.(*api.InstanceSnapshot))
}
// Add the ContainerBackups
backups, err := d.Backups()
if err != nil {
return nil, nil, err
}
for _, backup := range backups {
render := backup.Render()
if ct.Backups == nil {
ct.Backups = []api.InstanceBackup{}
}
ct.Backups = append(ct.Backups, *render)
}
return &ct, etag, nil
}
// renderState renders just the running state of the instance.
func (d *lxc) renderState(statusCode api.StatusCode, hostInterfaces []net.Interface) (*api.InstanceState, error) {
status := api.InstanceState{
Status: statusCode.String(),
StatusCode: statusCode,
}
// If container is in error state, we're done here.
if d.isErrorStatusCode(statusCode) {
return &status, nil
}
pid := d.InitPID()
processesState, _ := d.processesState(pid)
if d.isRunningStatusCode(statusCode) {
var err error
status.CPU = d.cpuState()
status.Memory = d.memoryState()
status.Network = d.networkState(hostInterfaces)
status.Pid = int64(pid)
status.Processes = processesState
status.StartedAt, err = d.processStartedAt(d.InitPID())
if err != nil {
return nil, err
}
}
status.Disk = d.diskState()
d.release()
return &status, nil
}
// RenderState renders just the running state of the instance.
func (d *lxc) RenderState(hostInterfaces []net.Interface) (*api.InstanceState, error) {
return d.renderState(d.statusCode(), hostInterfaces)
}
// snapshot creates a snapshot of the instance.
func (d *lxc) snapshot(name string, expiry time.Time, stateful bool) error {
// Check that migration.stateful is set for stateful actions.
if stateful && !d.CanLiveMigrate() {
return errors.New("Stateful snapshots require that the instance has migration.stateful be set to true")
}
// Deal with state.
if stateful {
// Quick checks.
if !d.IsRunning() {
return errors.New("Unable to create a stateful snapshot. The instance isn't running")
}
_, err := exec.LookPath("criu")
if err != nil {
return errors.New("Unable to create a stateful snapshot. CRIU isn't installed")
}
// Cleanup any existing state
stateDir := d.StatePath()
_ = os.RemoveAll(stateDir)
// Create the state path and make sure we don't keep state around after the snapshot has been made.
err = os.MkdirAll(stateDir, 0o700)
if err != nil {
return err
}
defer logger.WarnOnError(func() error { return os.RemoveAll(stateDir) }, "Failed to remove state directory")
// Release liblxc container once done.
defer func() {
d.release()
}()
// Load the go-lxc struct
if d.expandedConfig["raw.lxc"] != "" {
cc, err := d.initLXC(true)
if err != nil {
return err
}
err = d.loadRawLXCConfig(cc)
if err != nil {
return err
}
} else {
_, err = d.initLXC(false)
if err != nil {
return err
}
}
/* TODO: ideally we would freeze here and unfreeze below after
* we've copied the filesystem, to make sure there are no
* changes by the container while snapshotting. Unfortunately
* there is abug in CRIU where it doesn't leave the container
* in the same state it found it w.r.t. freezing, i.e. CRIU
* freezes too, and then /always/ thaws, even if the container
* was frozen. Until that's fixed, all calls to Unfreeze()
* after snapshotting will fail.
*/
criuMigrationArgs := instance.CriuMigrationArgs{
Cmd: liblxc.MIGRATE_DUMP,
StateDir: stateDir,
Function: "snapshot",
Stop: false,
ActionScript: false,
DumpDir: "",
PreDumpDir: "",
}
// Dump the state.
err = d.migrate(&criuMigrationArgs)
if err != nil {
return fmt.Errorf("Failed taking stateful checkpoint: %w", err)
}
}
// Wait for any file operations to complete to have a more consistent snapshot.
d.stopForkfile(false)
return d.snapshotCommon(d, name, expiry, stateful)
}
// Snapshot takes a new snapshot.
func (d *lxc) Snapshot(name string, expiry time.Time, stateful bool) error {
return d.snapshot(name, expiry, stateful)
}
// Restore restores a snapshot.
func (d *lxc) Restore(sourceContainer instance.Instance, stateful bool, diskOnly bool) error {
var ctxMap logger.Ctx
op, err := operationlock.Create(d.Project().Name, d.Name(), d.op, operationlock.ActionRestore, false, false)
if err != nil {
return fmt.Errorf("Failed to create instance restore operation: %w", err)
}
defer op.Done(nil)
// Initialize storage interface for the container.
pool, err := storagePools.LoadByInstance(d.state, d)
if err != nil {
op.Done(err)
return err
}
err = pool.CanRestoreInstanceSnapshot(d, sourceContainer)
if err != nil {
op.Done(err)
return err
}
// Stop the container.
wasRunning := d.IsRunning()
if wasRunning {
ephemeral := d.IsEphemeral()
if ephemeral {
// Unset ephemeral flag.
args := db.InstanceArgs{
Architecture: d.Architecture(),
Config: d.LocalConfig(),
Description: d.Description(),
Devices: d.LocalDevices(),
Ephemeral: false,
Profiles: d.Profiles(),
Project: d.Project().Name,
Type: d.Type(),
Snapshot: d.IsSnapshot(),
}
err := d.Update(args, false)
if err != nil {
op.Done(err)
return err
}
// On function return, set the flag back on.
defer func() {
args.Ephemeral = ephemeral
_ = d.Update(args, false)
}()
}
// This will unmount the container storage.
err := d.Stop(false)
if err != nil {
op.Done(err)
return err
}
// Refresh the operation as that one is now complete.
op, err = operationlock.Create(d.Project().Name, d.Name(), d.op, operationlock.ActionRestore, false, false)
if err != nil {
return fmt.Errorf("Failed to create instance restore operation: %w", err)
}
defer op.Done(nil)
}
ctxMap = logger.Ctx{
"created": d.creationDate,
"ephemeral": d.ephemeral,
"used": d.lastUsedDate,
"source": sourceContainer.Name(),
}
d.logger.Info("Restoring instance", ctxMap)
// Wait for any file operations to complete.
// This is required so we can actually unmount the container and restore its rootfs.
d.stopForkfile(false)
d.logger.Debug("Mounting instance to check for CRIU state path existence")
reverter := revert.New()
defer reverter.Fail()
// Ensure that storage is mounted for state path checks and for backup.yaml updates.
_, err = d.mount()
if err != nil {
op.Done(err)
return err
}
reverter.Add(func() { _ = d.unmount() })
// Check for CRIU if necessary, before doing a bunch of filesystem manipulations.
// Requires container be mounted to check StatePath exists.
if util.PathExists(d.StatePath()) {
_, err := exec.LookPath("criu")
if err != nil {
err = errors.New("Failed to restore container state. CRIU isn't installed")
op.Done(err)
return err
}
}
err = d.unmount()
if err != nil {
op.Done(err)
return err
}
reverter.Success()
// Restore the rootfs.
err = pool.RestoreInstanceSnapshot(d, sourceContainer, nil)
if err != nil {
op.Done(err)
return err
}
args := db.InstanceArgs{}
if !diskOnly {
// Restore the configuration.
args = db.InstanceArgs{
Architecture: sourceContainer.Architecture(),
Config: sourceContainer.LocalConfig(),
Description: sourceContainer.Description(),
Devices: sourceContainer.LocalDevices(),
Ephemeral: sourceContainer.IsEphemeral(),
Profiles: sourceContainer.Profiles(),
Project: sourceContainer.Project().Name,
Type: sourceContainer.Type(),
Snapshot: sourceContainer.IsSnapshot(),
}
} else {
args = db.InstanceArgs{
Architecture: d.Architecture(),
Config: d.LocalConfig(),
Description: d.Description(),
Devices: d.LocalDevices(),
Ephemeral: d.IsEphemeral(),
Profiles: d.Profiles(),
Project: d.Project().Name,
Type: d.Type(),
Snapshot: d.IsSnapshot(),
}
args.Config["volatile.uuid.generation"] = sourceContainer.LocalConfig()["volatile.uuid.generation"]
}
// Don't pass as user-requested as there's no way to fix a bad config.
// This will call d.UpdateBackupFile() to ensure snapshot list is up to date.
err = d.Update(args, false)
if err != nil {
op.Done(err)
return err
}
// If the container wasn't running but was stateful, should we restore it as running?
if stateful {
if !util.PathExists(d.StatePath()) {
err = errors.New("Stateful snapshot restore requested but snapshot is stateless")
op.Done(err)
return err
}
d.logger.Debug("Performing stateful restore", ctxMap)
d.stateful = true
criuMigrationArgs := instance.CriuMigrationArgs{
Cmd: liblxc.MIGRATE_RESTORE,
StateDir: d.StatePath(),
Function: "snapshot",
Stop: false,
ActionScript: false,
DumpDir: "",
PreDumpDir: "",
}
// Checkpoint.
err = d.migrate(&criuMigrationArgs)
if err != nil {
op.Done(err)
return fmt.Errorf("Failed taking stateful checkpoint: %w", err)
}
// Remove the state from the parent container; we only keep this in snapshots.
err2 := os.RemoveAll(d.StatePath())
if err2 != nil && !errors.Is(err, fs.ErrNotExist) {
op.Done(err)
return err
}
if err != nil {
op.Done(err)
return err
}
d.logger.Debug("Performed stateful restore", ctxMap)
d.logger.Info("Restored instance", ctxMap)
return nil
}
// Restart the container.
if wasRunning {
d.logger.Debug("Starting instance after snapshot restore")
err = d.Start(false)
if err != nil {
op.Done(err)
return err
}
}
d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceRestored.Event(d, map[string]any{"snapshot": sourceContainer.Name()}))
d.logger.Info("Restored instance", ctxMap)
return nil
}
func (d *lxc) cleanup() {
// Unmount any leftovers
_ = d.removeUnixDevices()
_ = d.removeDiskDevices()
// Remove the security profiles
_ = apparmor.InstanceDelete(d.state.OS, d)
seccomp.DeleteProfile(d)
// Remove the devices path
_ = os.Remove(d.DevicesPath())
// Remove the shmounts path
_ = os.RemoveAll(d.ShmountsPath())
}
// Delete deletes the instance.
// cleanupDependencies controls whether dependent resources (e.g. volumes,
// and related state) are removed along with the instance.
// When false, dependencies are preserved (e.g. storage-only moves).
func (d *lxc) Delete(force bool, cleanupDependencies bool) error {
// Setup a new operation.
op, err := operationlock.CreateWaitGet(d.Project().Name, d.Name(), d.op, operationlock.ActionDelete, nil, false, false)
if err != nil {
return fmt.Errorf("Failed to create instance delete operation: %w", err)
}
defer op.Done(nil)
if d.IsRunning() {
return api.StatusErrorf(http.StatusBadRequest, "Instance is running")
}
err = d.delete(force, cleanupDependencies)
if err != nil {
return err
}
// If dealing with a snapshot, refresh the backup file on the parent.
if d.IsSnapshot() {
parentName, _, _ := api.GetParentAndSnapshotName(d.name)
// Load the parent.
parent, err := instance.LoadByProjectAndName(d.state, d.project.Name, parentName)
if err != nil {
return fmt.Errorf("Invalid parent: %w", err)
}
// Update the backup file.
err = parent.UpdateBackupFile()
if err != nil {
return err
}
}
return nil
}
// Delete deletes the instance without creating an operation lock.
func (d *lxc) delete(force bool, cleanupDependencies bool) error {
ctxMap := logger.Ctx{
"created": d.creationDate,
"ephemeral": d.ephemeral,
"used": d.lastUsedDate,
}
if d.isSnapshot {
d.logger.Info("Deleting instance snapshot", ctxMap)
} else {
d.logger.Info("Deleting instance", ctxMap)
}
if !force && util.IsTrue(d.expandedConfig["security.protection.delete"]) && !d.IsSnapshot() {
err := errors.New("Instance is protected")
d.logger.Warn("Failed to delete instance", logger.Ctx{"err": err})
return err
}
// Wait for any file operations to complete.
// This is required so we can actually unmount the container and delete it.
if !d.IsSnapshot() {
d.stopForkfile(false)
}
// Delete any persistent warnings for instance.
err := d.warningsDelete()
if err != nil {
return err
}
pool, err := storagePools.LoadByInstance(d.state, d)
if err != nil && !response.IsNotFoundError(err) {
return err
} else if pool != nil {
if d.IsSnapshot() {
// Remove snapshot volume and database record.
err = pool.DeleteInstanceSnapshot(d, nil)
if err != nil {
return err
}
} else {
// Remove all snapshots.
err := d.deleteSnapshots(func(snapInst instance.Instance) error {
return snapInst.(*lxc).delete(true, cleanupDependencies) // Internal delete function that doesn't lock.
})
if err != nil {
return fmt.Errorf("Failed deleting instance snapshots: %w", err)
}
// Remove the storage volume and database records.
err = pool.DeleteInstance(d, nil)
if err != nil {
return err
}
if cleanupDependencies {
// Delete all dependent volumes associated with this instance.
err = d.ForEachDependentDiskType(func(dev deviceConfig.DeviceNamed) error {
// Load the pool for the disk.
diskPool, err := storagePools.LoadByName(d.state, dev.Config["pool"])
if err != nil {
return fmt.Errorf("Failed loading storage pool: %w", err)
}
volName, _ := internalInstance.SplitVolumeSource(dev.Config["source"])
err = diskPool.DeleteCustomVolume(d.Project().Name, volName, nil)
if err != nil {
return err
}
return nil
})
if err != nil {
return err
}
}
}
}
// Perform other cleanup steps if not snapshot.
if !d.IsSnapshot() {
// Remove all backups.
backups, err := d.Backups()
if err != nil {
return err
}
for _, backup := range backups {
err = backup.Delete()
if err != nil {
return err
}
}
// Run device removal function for each device.
d.devicesRemove(d, cleanupDependencies)
// Clean things up.
d.cleanup()
}
err = d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error {
// Remove the database record of the instance or snapshot instance.
return tx.DeleteInstance(ctx, d.project.Name, d.Name())
})
if err != nil {
d.logger.Error("Failed deleting instance entry", logger.Ctx{"err": err})
return err
}
if d.isSnapshot {
d.logger.Info("Deleted instance snapshot", ctxMap)
} else {
d.logger.Info("Deleted instance", ctxMap)
}
if d.isSnapshot {
d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceSnapshotDeleted.Event(d, nil))
} else {
err = d.state.Authorizer.DeleteInstance(d.state.ShutdownCtx, d.project.Name, d.Name())
if err != nil {
logger.Error("Failed to remove instance from authorizer", logger.Ctx{"name": d.Name(), "project": d.project.Name, "error": err})
}
d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceDeleted.Event(d, nil))
}
return nil
}
// Rename renames the instance. Accepts an argument to enable applying deferred TemplateTriggerRename.
func (d *lxc) Rename(newName string, applyTemplateTrigger bool) error {
oldName := d.Name()
ctxMap := logger.Ctx{
"created": d.creationDate,
"ephemeral": d.ephemeral,
"used": d.lastUsedDate,
"newname": newName,
}
d.logger.Info("Renaming instance", ctxMap)
// Quick checks.
err := instance.ValidName(newName, d.IsSnapshot())
if err != nil {
return err
}
if d.IsRunning() {
return errors.New("Renaming of running instance not allowed")
}
// Clean things up.
d.cleanup()
pool, err := storagePools.LoadByInstance(d.state, d)
if err != nil {
return fmt.Errorf("Failed loading instance storage pool: %w", err)
}
if d.IsSnapshot() {
_, newSnapName, _ := api.GetParentAndSnapshotName(newName)
err = pool.RenameInstanceSnapshot(d, newSnapName, nil)
if err != nil {
return fmt.Errorf("Rename instance snapshot: %w", err)
}
} else {
err = pool.RenameInstance(d, newName, nil)
if err != nil {
return fmt.Errorf("Rename instance: %w", err)
}
if applyTemplateTrigger {
err = d.DeferTemplateApply(instance.TemplateTriggerRename)
if err != nil {
return err
}
}
}
if !d.IsSnapshot() {
var results []string
err := d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error {
var err error
// Rename all the instance snapshot database entries.
results, err = tx.GetInstanceSnapshotsNames(ctx, d.project.Name, oldName)
if err != nil {
d.logger.Error("Failed to get instance snapshots", ctxMap)
return fmt.Errorf("Failed to get instance snapshots: Failed getting instance snapshot names: %w", err)
}
for _, sname := range results {
// Rename the snapshot.
oldSnapName := strings.SplitN(sname, internalInstance.SnapshotDelimiter, 2)[1]
baseSnapName := filepath.Base(sname)
err := cluster.RenameInstanceSnapshot(ctx, tx.Tx(), d.project.Name, oldName, oldSnapName, baseSnapName)
if err != nil {
d.logger.Error("Failed renaming snapshot", ctxMap)
return fmt.Errorf("Failed renaming snapshot: %w", err)
}
}
return nil
})
if err != nil {
return err
}
}
// Rename the instance database entry.
err = d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error {
if d.IsSnapshot() {
oldParts := strings.SplitN(oldName, internalInstance.SnapshotDelimiter, 2)
newParts := strings.SplitN(newName, internalInstance.SnapshotDelimiter, 2)
return cluster.RenameInstanceSnapshot(ctx, tx.Tx(), d.project.Name, oldParts[0], oldParts[1], newParts[1])
}
return cluster.RenameInstance(ctx, tx.Tx(), d.project.Name, oldName, newName)
})
if err != nil {
d.logger.Error("Failed renaming instance", ctxMap)
return fmt.Errorf("Failed renaming instance: %w", err)
}
// Rename the logging path.
newFullName := project.Instance(d.Project().Name, d.Name())
_ = os.RemoveAll(internalUtil.LogPath(newFullName))
if util.PathExists(d.LogPath()) {
err := os.Rename(d.LogPath(), internalUtil.LogPath(newFullName))
if err != nil {
d.logger.Error("Failed renaming instance", ctxMap)
return fmt.Errorf("Failed renaming instance: %w", err)
}
}
// Rename the runtime path.
newFullName = project.Instance(d.Project().Name, d.Name())
_ = os.RemoveAll(internalUtil.RunPath(newFullName))
if util.PathExists(d.RunPath()) {
err := os.Rename(d.RunPath(), internalUtil.RunPath(newFullName))
if err != nil {
d.logger.Error("Failed renaming instance", ctxMap)
return fmt.Errorf("Failed renaming instance: %w", err)
}
}
reverter := revert.New()
defer reverter.Fail()
// Set the new name in the struct.
d.name = newName
reverter.Add(func() { d.name = oldName })
// Rename the backups.
backups, err := d.Backups()
if err != nil {
return err
}
for _, backup := range backups {
b := backup
oldName := b.Name()
backupName := strings.Split(oldName, "/")[1]
newName := fmt.Sprintf("%s/%s", newName, backupName)
err = b.Rename(newName)
if err != nil {
return err
}
reverter.Add(func() { _ = b.Rename(oldName) })
}
// Invalidate the go-lxc cache.
d.release()
d.cConfig = false
// Update lease files.
err = network.UpdateDNSMasqStatic(d.state, "")
if err != nil {
return err
}
// Reset cloud-init instance-id (causes a re-run on name changes).
if !d.IsSnapshot() {
err = d.resetInstanceID()
if err != nil {
return err
}
}
// Update the backup file.
err = d.UpdateBackupFile()
if err != nil {
return err
}
d.logger.Info("Renamed instance", ctxMap)
if d.isSnapshot {
d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceSnapshotRenamed.Event(d, map[string]any{"old_name": oldName}))
} else {
err = d.state.Authorizer.RenameInstance(d.state.ShutdownCtx, d.project.Name, oldName, newName)
if err != nil {
logger.Error("Failed to rename instance in authorizer", logger.Ctx{"old_name": oldName, "new_name": newName, "project": d.project.Name, "error": err})
}
d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceRenamed.Event(d, map[string]any{"old_name": oldName}))
}
reverter.Success()
return nil
}
// CGroupSet sets a cgroup value for the instance.
func (d *lxc) CGroupSet(key string, value string) error {
// Load the go-lxc struct
cc, err := d.initLXC(false)
if err != nil {
return err
}
// Make sure the container is running.
// We use InitPID here rather than IsRunning because this task can be triggered during the container's
// startup process, which is during the time that the start lock is held, which causes IsRunning to
// return false (because the container hasn't fully started yet) but it is sufficiently started to
// have its cgroup disk limits set.
if d.InitPID() <= 0 {
return errors.New("Can't set cgroups on a stopped container")
}
err = cc.SetCgroupItem(key, value)
if err != nil {
return fmt.Errorf("Failed to set cgroup %s=\"%s\": %w", key, value, err)
}
return nil
}
// Update applies updated config.
func (d *lxc) Update(args db.InstanceArgs, userRequested bool) error {
// Setup a new operation
op, err := operationlock.CreateWaitGet(d.Project().Name, d.Name(), d.op, operationlock.ActionUpdate, []operationlock.Action{operationlock.ActionCreate, operationlock.ActionRestart, operationlock.ActionRestore}, false, false)
if err != nil {
return fmt.Errorf("Failed to create instance update operation: %w", err)
}
defer op.Done(nil)
// Set sane defaults for unset keys
if args.Project == "" {
args.Project = api.ProjectDefaultName
}
if args.Architecture == 0 {
args.Architecture = d.architecture
}
if args.Config == nil {
args.Config = map[string]string{}
}
if args.Devices == nil {
args.Devices = deviceConfig.Devices{}
}
if args.Profiles == nil {
args.Profiles = []api.Profile{}
}
if userRequested {
// Validate the new config
err := instance.ValidConfig(d.state.OS, args.Config, false, d.dbType)
if err != nil {
return fmt.Errorf("Invalid config: %w", err)
}
// Validate the new devices without using expanded devices validation (expensive checks disabled).
err = instance.ValidDevices(d.state, d.project, d.Type(), args.Devices, nil)
if err != nil {
return fmt.Errorf("Invalid devices: %w", err)
}
}
var profiles []string
err = d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error {
// Validate the new profiles
profiles, err = tx.GetProfileNames(ctx, args.Project)
return err
})
if err != nil {
return fmt.Errorf("Failed to get profiles: %w", err)
}
checkedProfiles := []string{}
for _, profile := range args.Profiles {
if !slices.Contains(profiles, profile.Name) {
return fmt.Errorf("Requested profile '%s' doesn't exist", profile.Name)
}
if slices.Contains(checkedProfiles, profile.Name) {
return errors.New("Duplicate profile found in request")
}
checkedProfiles = append(checkedProfiles, profile.Name)
}
// Validate the new architecture
if args.Architecture != 0 {
_, err = osarch.ArchitectureName(args.Architecture)
if err != nil {
return fmt.Errorf("Invalid architecture id: %s", err)
}
}
// Get a copy of the old configuration
oldDescription := d.Description()
oldArchitecture := 0
err = util.DeepCopy(&d.architecture, &oldArchitecture)
if err != nil {
return err
}
oldEphemeral := false
err = util.DeepCopy(&d.ephemeral, &oldEphemeral)
if err != nil {
return err
}
oldExpandedDevices := deviceConfig.Devices{}
err = util.DeepCopy(&d.expandedDevices, &oldExpandedDevices)
if err != nil {
return err
}
oldExpandedConfig := map[string]string{}
err = util.DeepCopy(&d.expandedConfig, &oldExpandedConfig)
if err != nil {
return err
}
oldLocalDevices := deviceConfig.Devices{}
err = util.DeepCopy(&d.localDevices, &oldLocalDevices)
if err != nil {
return err
}
oldLocalConfig := map[string]string{}
err = util.DeepCopy(&d.localConfig, &oldLocalConfig)
if err != nil {
return err
}
oldProfiles := []api.Profile{}
err = util.DeepCopy(&d.profiles, &oldProfiles)
if err != nil {
return err
}
oldExpiryDate := d.expiryDate
// Define a function which reverts everything. Defer this function
// so that it doesn't need to be explicitly called in every failing
// return path. Track whether or not we want to undo the changes
// using a closure.
undoChanges := true
defer func() {
if undoChanges {
d.description = oldDescription
d.architecture = oldArchitecture
d.ephemeral = oldEphemeral
d.expandedConfig = oldExpandedConfig
d.expandedDevices = oldExpandedDevices
d.localConfig = oldLocalConfig
d.localDevices = oldLocalDevices
d.profiles = oldProfiles
d.expiryDate = oldExpiryDate
d.release()
d.cConfig = false
_, _ = d.initLXC(true)
_ = d.setupCredentials(true)
cgroup.TaskSchedulerTrigger("container", d.name, "changed")
}
}()
// Apply the various changes
d.description = args.Description
d.architecture = args.Architecture
d.ephemeral = args.Ephemeral
d.localConfig = args.Config
d.localDevices = args.Devices
d.profiles = args.Profiles
d.expiryDate = args.ExpiryDate
// Expand the config and refresh the LXC config
err = d.expandConfig()
if err != nil {
return err
}
// Diff the configurations
changedConfig := []string{}
for key := range oldExpandedConfig {
if oldExpandedConfig[key] != d.expandedConfig[key] {
if !slices.Contains(changedConfig, key) {
changedConfig = append(changedConfig, key)
}
}
}
for key := range d.expandedConfig {
if oldExpandedConfig[key] != d.expandedConfig[key] {
if !slices.Contains(changedConfig, key) {
changedConfig = append(changedConfig, key)
}
}
}
// Diff the devices
removeDevices, addDevices, updateDevices, allUpdatedKeys := oldExpandedDevices.Update(d.expandedDevices, func(oldDevice deviceConfig.Device, newDevice deviceConfig.Device) []string {
// This function needs to return a list of fields that are excluded from differences
// between oldDevice and newDevice. The result of this is that as long as the
// devices are otherwise identical except for the fields returned here, then the
// device is considered to be being "updated" rather than "added & removed".
oldDevType, err := device.LoadByType(d.state, d.Project().Name, oldDevice)
if err != nil {
return []string{} // Couldn't create Device, so this cannot be an update.
}
newDevType, err := device.LoadByType(d.state, d.Project().Name, newDevice)
if err != nil {
return []string{} // Couldn't create Device, so this cannot be an update.
}
// Detached devices need to be fully recreated on update so that the update logic doesn't
// try to access non-existing LXC devices.
if !util.IsTrueOrEmpty(oldDevice["attached"]) {
return []string{}
}
return newDevType.UpdatableFields(oldDevType)
})
// Prevent adding or updating device initial configuration.
if util.StringPrefixInSlice("initial.", allUpdatedKeys) {
for devName, newDev := range addDevices {
for k, newVal := range newDev {
if !strings.HasPrefix(k, "initial.") {
continue
}
if strings.HasPrefix(newDev["source"], "tmpfs:") || strings.HasPrefix(newDev["source"], "tmpfs-overlay:") {
continue
}
if newDev["pool"] != "" && newDev["path"] != "/" && strings.Contains(newDev["source"], "/") {
continue
}
oldDev, ok := removeDevices[devName]
if !ok {
return errors.New("New device with initial configuration cannot be added once the instance is created")
}
oldVal, ok := oldDev[k]
if !ok {
return errors.New("Device initial configuration cannot be added once the instance is created")
}
// If newVal is an empty string it means the initial configuration
// has been removed.
if newVal != "" && newVal != oldVal {
return errors.New("Device initial configuration cannot be modified once the instance is created")
}
}
}
}
if userRequested {
// Look for deleted idmap keys.
protectedKeys := []string{
"volatile.idmap.base",
"volatile.idmap.current",
"volatile.idmap.next",
"volatile.last_state.idmap",
}
for _, k := range changedConfig {
if !slices.Contains(protectedKeys, k) {
continue
}
_, ok := d.expandedConfig[k]
if !ok {
return errors.New("Volatile idmap keys can't be deleted by the user")
}
}
// Do some validation of the config diff (allows mixed instance types for profiles).
err = instance.ValidConfig(d.state.OS, d.expandedConfig, true, instancetype.Any)
if err != nil {
return fmt.Errorf("Invalid expanded config: %w", err)
}
// Do full expanded validation of the devices diff.
err = instance.ValidDevices(d.state, d.project, d.Type(), d.localDevices, d.expandedDevices)
if err != nil {
return fmt.Errorf("Invalid expanded devices: %w", err)
}
// Validate root device
_, oldRootDev, oldErr := internalInstance.GetRootDiskDevice(oldExpandedDevices.CloneNative())
_, newRootDev, newErr := internalInstance.GetRootDiskDevice(d.expandedDevices.CloneNative())
if oldErr == nil && newErr == nil && oldRootDev["pool"] != newRootDev["pool"] {
return fmt.Errorf("Cannot update root disk device pool name to %q", newRootDev["pool"])
}
// Ensure the instance has a root disk.
if newErr != nil {
return fmt.Errorf("Invalid root disk device: %w", newErr)
}
}
// Run through initLXC to catch anything we missed
if userRequested {
d.release()
d.cConfig = false
_, err = d.initLXC(true)
if err != nil {
return fmt.Errorf("Initialize LXC: %w", err)
}
}
// If raw.lxc changed, re-validate the config.
if slices.Contains(changedConfig, "raw.lxc") && d.expandedConfig["raw.lxc"] != "" {
// Get a new liblxc instance.
cc, err := liblxc.NewContainer(d.name, d.state.OS.LxcPath)
if err != nil {
return err
}
err = d.loadRawLXCConfig(cc)
if err != nil {
// Release the liblxc instance.
_ = cc.Release()
return err
}
// Release the liblxc instance.
_ = cc.Release()
}
// If apparmor changed, re-validate the apparmor profile (even if not running).
if slices.Contains(changedConfig, "raw.apparmor") || slices.Contains(changedConfig, "security.nesting") {
err = apparmor.InstanceValidate(d.state.OS, d, nil)
if err != nil {
return fmt.Errorf("Parse AppArmor profile: %w", err)
}
}
if slices.Contains(changedConfig, "security.idmap.isolated") || slices.Contains(changedConfig, "security.idmap.base") || slices.Contains(changedConfig, "security.idmap.size") || slices.Contains(changedConfig, "raw.idmap") || slices.Contains(changedConfig, "security.privileged") {
var idmapSet *idmap.Set
base := int64(0)
if !d.IsPrivileged() {
// Update the idmap.
idmapSet, base, err = d.findIdmap()
if err != nil {
return fmt.Errorf("Failed to get ID map: %w", err)
}
}
jsonIdmap, err := idmapSet.ToJSON()
if err != nil {
return fmt.Errorf("Failed to encode ID map: %w", err)
}
d.localConfig["volatile.idmap.next"] = jsonIdmap
d.localConfig["volatile.idmap.base"] = fmt.Sprintf("%v", base)
// Invalidate the idmap cache.
d.idmapset = nil
}
isRunning := d.IsRunning()
// Use the device interface to apply update changes.
err = d.devicesUpdate(d, removeDevices, addDevices, updateDevices, oldExpandedDevices, isRunning, userRequested)
if err != nil {
return err
}
// Apply the live changes
if isRunning {
cc, err := d.initLXC(false)
if err != nil {
return err
}
cg, err := d.cgroup(cc, true)
if err != nil {
return err
}
// Live update the container config
for _, key := range changedConfig {
value := d.expandedConfig[key]
if key == "raw.apparmor" || key == "security.nesting" {
// Update the AppArmor profile
err = apparmor.InstanceLoad(d.state.OS, d, nil)
if err != nil {
return err
}
} else if key == "security.guestapi" {
if util.IsTrueOrEmpty(value) {
err = d.insertMount(internalUtil.VarPath("guestapi"), "/dev/incus", "none", unix.MS_BIND, idmap.StorageTypeNone)
if err != nil {
return err
}
} else {
err = func() error {
// Connect to files API.
files, err := d.FileSFTP()
if err != nil {
return err
}
defer logger.WarnOnError(files.Close, "Failed to close SFTP connection")
_, err = files.Lstat("/dev/incus")
if err == nil {
err = d.removeMount("/dev/incus")
if err != nil {
return err
}
err = files.Remove("/dev/incus")
if err != nil {
return err
}
}
return nil
}()
if err != nil {
return err
}
}
} else if key == "linux.kernel_modules" && value != "" {
for _, module := range strings.Split(value, ",") {
module = strings.TrimPrefix(module, " ")
err := linux.LoadModule(module)
if err != nil {
return fmt.Errorf("Failed to load kernel module '%s': %w", module, err)
}
}
} else if key == "limits.disk.priority" {
if !cgroup.Supports(cgroup.IO) {
continue
}
priorityInt := 5
diskPriority := d.expandedConfig["limits.disk.priority"]
if diskPriority != "" {
priorityInt, err = strconv.Atoi(diskPriority)
if err != nil {
return err
}
}
// Minimum valid value is 10
priority := int64(priorityInt * 100)
if priority == 0 {
priority = 10
}
err = cg.SetBlkioWeight(priority)
if err != nil {
return err
}
} else if key == "limits.memory.oom_priority" {
// Configure the OOM priority.
err = d.setOOMPriority(cc.InitPid())
if err != nil {
d.logger.Warn("Failed to set OOM priority", logger.Ctx{
"err": err,
"instance": d.Name(),
"project": d.Project().Name,
})
}
} else if key == "limits.memory" || strings.HasPrefix(key, "limits.memory.") {
// Skip if no memory CGroup
if !cgroup.Supports(cgroup.Memory) {
continue
}
// Set the new memory limit
memory := d.expandedConfig["limits.memory"]
memoryEnforce := d.expandedConfig["limits.memory.enforce"]
memorySwap := d.expandedConfig["limits.memory.swap"]
var memoryInt int64
// Parse memory
if memory == "" {
memoryInt = -1
} else {
memoryInt, err = ParseMemoryStr(memory)
if err != nil {
return err
}
}
// Store the old values for revert
oldMemswLimit := int64(-1)
if cgroup.Supports(cgroup.MemorySwap) {
oldMemswLimit, err = cg.GetMemorySwapLimit()
if err != nil {
oldMemswLimit = -1
}
}
oldLimit, err := cg.GetMemoryLimit()
if err != nil {
oldLimit = -1
}
oldSoftLimit, err := cg.GetMemorySoftLimit()
if err != nil {
oldSoftLimit = -1
}
revertMemory := func() {
if oldSoftLimit != -1 {
_ = cg.SetMemorySoftLimit(oldSoftLimit)
}
if oldLimit != -1 {
_ = cg.SetMemoryLimit(oldLimit)
}
if oldMemswLimit != -1 {
_ = cg.SetMemorySwapLimit(oldMemswLimit)
}
}
// Reset everything
if cgroup.Supports(cgroup.MemorySwap) {
err = cg.SetMemorySwapLimit(-1)
if err != nil {
revertMemory()
return err
}
}
err = cg.SetMemoryLimit(-1)
if err != nil {
revertMemory()
return err
}
err = cg.SetMemorySoftLimit(-1)
if err != nil {
revertMemory()
return err
}
// Set the new values
if memoryEnforce == "soft" {
// Set new limit.
err = cg.SetMemorySoftLimit(memoryInt)
if err != nil {
revertMemory()
return err
}
} else {
err = cg.SetMemoryLimit(memoryInt)
if err != nil {
revertMemory()
return err
}
if util.IsTrueOrEmpty(memorySwap) || util.IsFalse(memorySwap) {
err = cg.SetMemorySwapLimit(0)
if err != nil {
// Ignore missing swap accounting unless explicitly configured.
if memorySwap != "" || !errors.Is(err, cgroup.ErrControllerMissing) {
revertMemory()
return err
}
}
} else {
// Additional memory as swap.
swapInt, err := units.ParseByteSizeString(memorySwap)
if err != nil {
revertMemory()
return err
}
err = cg.SetMemorySwapLimit(swapInt)
if err != nil {
revertMemory()
return err
}
}
}
// Configure the swappiness
if key == "limits.memory.swap" || key == "limits.memory.swap.priority" {
memorySwapPriority := d.expandedConfig["limits.memory.swap.priority"]
if util.IsFalse(memorySwap) {
err = cg.SetMemorySwappiness(0)
if err != nil && !errors.Is(err, cgroup.ErrControllerMissing) {
return err
}
} else {
priority := 10
if memorySwapPriority != "" {
priority, err = strconv.Atoi(memorySwapPriority)
if err != nil {
return err
}
}
// Maximum priority (10) should be default swappiness (60).
err = cg.SetMemorySwappiness(int64(70 - priority))
if err != nil {
// Ignore missing swappiness support unless explicitly configured.
if memorySwapPriority != "" || !errors.Is(err, cgroup.ErrControllerMissing) {
return err
}
}
}
}
} else if key == "limits.cpu" || key == "limits.cpu.nodes" {
// Clear the "volatile.cpu.nodes" if needed.
d.ClearLimitsCPUNodes(changedConfig)
// Trigger a scheduler re-run
defer cgroup.TaskSchedulerTrigger("container", d.name, "changed") //nolint:revive
} else if key == "limits.cpu.priority" || key == "limits.cpu.allowance" {
// Skip if no cpu CGroup
if !cgroup.Supports(cgroup.CPU) {
continue
}
// Apply new CPU limits
cpuShares, cpuCfsQuota, cpuCfsPeriod, err := cgroup.ParseCPU(d.expandedConfig["limits.cpu.allowance"], d.expandedConfig["limits.cpu.priority"])
if err != nil {
return err
}
err = cg.SetCPUShare(cpuShares)
if err != nil {
return err
}
err = cg.SetCPUCfsLimit(cpuCfsPeriod, cpuCfsQuota)
if err != nil {
return err
}
} else if key == "limits.processes" {
if !cgroup.Supports(cgroup.Pids) {
continue
}
if value == "" {
err = cg.SetMaxProcesses(-1)
if err != nil {
return err
}
} else {
valueInt, err := strconv.ParseInt(value, 10, 64)
if err != nil {
return err
}
err = cg.SetMaxProcesses(valueInt)
if err != nil {
return err
}
}
} else if strings.HasPrefix(key, "limits.hugepages.") {
if !cgroup.Supports(cgroup.Hugetlb) {
continue
}
pageType := ""
switch key {
case "limits.hugepages.64KB":
pageType = "64KB"
case "limits.hugepages.1MB":
pageType = "1MB"
case "limits.hugepages.2MB":
pageType = "2MB"
case "limits.hugepages.1GB":
pageType = "1GB"
}
valueInt := int64(-1)
if value != "" {
valueInt, err = units.ParseByteSizeString(value)
if err != nil {
return err
}
}
err = cg.SetHugepagesLimit(pageType, valueInt)
if err != nil {
return err
}
}
}
// Update the credentials directory.
err = d.setupCredentials(true)
if err != nil {
return err
}
}
// Re-generate the instance-id if needed.
if !d.IsSnapshot() && d.needsNewInstanceID(changedConfig, oldExpandedDevices) {
err = d.resetInstanceID()
if err != nil {
return err
}
}
// Finally, apply the changes to the database
err = d.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error {
// Snapshots should update only their descriptions and expiry date.
if d.IsSnapshot() {
return tx.UpdateInstanceSnapshot(d.id, d.description, d.expiryDate)
}
object, err := cluster.GetInstance(ctx, tx.Tx(), d.project.Name, d.name)
if err != nil {
return err
}
object.Description = d.description
object.Architecture = d.architecture
object.Ephemeral = d.ephemeral
object.ExpiryDate = sql.NullTime{Time: d.expiryDate, Valid: true}
err = cluster.UpdateInstance(ctx, tx.Tx(), d.project.Name, d.name, *object)
if err != nil {
return err
}
err = cluster.UpdateInstanceConfig(ctx, tx.Tx(), int64(object.ID), d.localConfig)
if err != nil {
return err
}
devices, err := cluster.APIToDevices(d.localDevices.CloneNative())
if err != nil {
return err
}
err = cluster.UpdateInstanceDevices(ctx, tx.Tx(), int64(object.ID), devices)
if err != nil {
return err
}
profileNames := make([]string, 0, len(d.profiles))
for _, profile := range d.profiles {
profileNames = append(profileNames, profile.Name)
}
return cluster.UpdateInstanceProfiles(ctx, tx.Tx(), object.ID, object.Project, profileNames)
})
if err != nil {
return fmt.Errorf("Failed to update database: %w", err)
}
err = d.UpdateBackupFile()
if err != nil && !errors.Is(err, fs.ErrNotExist) {
return fmt.Errorf("Failed to write backup file: %w", err)
}
// Send devIncus notifications
if isRunning {
// Config changes (only for user.* keys
for _, key := range changedConfig {
if !strings.HasPrefix(key, "user.") {
continue
}
msg := map[string]any{
"key": key,
"old_value": oldExpandedConfig[key],
"value": d.expandedConfig[key],
}
err = d.devIncusEventSend("config", msg)
if err != nil {
return err
}
}
// Device changes
for k, m := range removeDevices {
msg := map[string]any{
"action": "removed",
"name": k,
"config": m,
}
err = d.devIncusEventSend("device", msg)
if err != nil {
return err
}
}
for k, m := range updateDevices {
msg := map[string]any{
"action": "updated",
"name": k,
"config": m,
}
err = d.devIncusEventSend("device", msg)
if err != nil {
return err
}
}
for k, m := range addDevices {
msg := map[string]any{
"action": "added",
"name": k,
"config": m,
}
err = d.devIncusEventSend("device", msg)
if err != nil {
return err
}
}
}
// Success, update the closure to mark that the changes should be kept.
undoChanges = false
if userRequested {
if d.isSnapshot {
d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceSnapshotUpdated.Event(d, nil))
} else {
d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceUpdated.Event(d, nil))
}
}
return nil
}
// Export backs up the instance.
func (d *lxc) Export(metaWriter io.Writer, rootfsWriter io.Writer, properties map[string]string, expiration time.Time, tracker *ioprogress.ProgressTracker) (*api.ImageMetadata, error) {
ctxMap := logger.Ctx{
"created": d.creationDate,
"ephemeral": d.ephemeral,
"used": d.lastUsedDate,
}
if d.IsRunning() {
return nil, errors.New("Cannot export a running instance as an image")
}
d.logger.Info("Exporting instance", ctxMap)
// Start the storage.
_, err := d.mount()
if err != nil {
d.logger.Error("Failed exporting instance", ctxMap)
return nil, err
}
defer logger.WarnOnError(d.unmount, "Failed to unmount instance")
// Get IDMap to unshift container as the tarball is created.
diskIdmap, err := d.DiskIdmap()
if err != nil {
d.logger.Error("Failed exporting instance", ctxMap)
return nil, err
}
// Create the tarball.
metaTarWriter := instancewriter.NewInstanceTarWriter(metaWriter, diskIdmap)
var rootfsTarWriter *instancewriter.InstanceTarWriter
if rootfsWriter != nil {
rootfsTarWriter = instancewriter.NewInstanceTarWriter(rootfsWriter, diskIdmap)
}
// Keep track of the first path we saw for each path with nlink>1.
cDir := d.Path()
// Path inside the tar image is the pathname starting after cDir.
// For the rootfs tarball in a split image, the path inside is the pathname starting after rootfs/
metaOffset := len(cDir) + 1
rootfsOffset := len(d.RootfsPath())
writeToMetaTar := func(fPath string, fi os.FileInfo, err error) error {
if err != nil {
return err
}
err = metaTarWriter.WriteFile(fPath[metaOffset:], fPath, fi, false)
if err != nil {
d.logger.Debug("Error tarring up", logger.Ctx{"path": fPath, "err": err})
return err
}
return nil
}
var writeToRootfsTar func(string, os.FileInfo, error) error
if rootfsWriter != nil {
writeToRootfsTar = func(path string, fi os.FileInfo, err error) error {
if err != nil {
return err
}
err = rootfsTarWriter.WriteFile(path[rootfsOffset:], path, fi, false)
if err != nil {
d.logger.Debug("Error tarring up", logger.Ctx{"path": path, "err": err})
return err
}
return nil
}
}
// Get the instance's architecture.
var arch string
if d.IsSnapshot() {
parentName, _, _ := api.GetParentAndSnapshotName(d.name)
parent, err := instance.LoadByProjectAndName(d.state, d.project.Name, parentName)
if err != nil {
_ = metaTarWriter.Close()
if rootfsTarWriter != nil {
_ = rootfsTarWriter.Close()
}
d.logger.Error("Failed exporting instance", ctxMap)
return nil, err
}
arch, _ = osarch.ArchitectureName(parent.Architecture())
} else {
arch, _ = osarch.ArchitectureName(d.architecture)
}
if arch == "" {
arch, err = osarch.ArchitectureName(d.state.OS.Architectures[0])
if err != nil {
d.logger.Error("Failed exporting instance", ctxMap)
return nil, err
}
}
// Generate metadata.yaml.
meta := api.ImageMetadata{}
fnam := filepath.Join(cDir, "metadata.yaml")
if util.PathExists(fnam) {
// Parse the metadata.
content, err := os.ReadFile(fnam)
if err != nil {
_ = metaTarWriter.Close()
if rootfsTarWriter != nil {
_ = rootfsTarWriter.Close()
}
d.logger.Error("Failed exporting instance", ctxMap)
return nil, err
}
err = yaml.Load(content, &meta)
if err != nil {
_ = metaTarWriter.Close()
if rootfsTarWriter != nil {
_ = rootfsTarWriter.Close()
}
d.logger.Error("Failed exporting instance", ctxMap)
return nil, err
}
}
// Fill in the metadata.
meta.Architecture = arch
meta.CreationDate = time.Now().UTC().Unix()
if meta.Properties == nil {
meta.Properties = map[string]string{}
}
maps.Copy(meta.Properties, properties)
if !expiration.IsZero() {
meta.ExpiryDate = expiration.UTC().Unix()
}
// Write the new metadata.yaml.
tempDir, err := os.MkdirTemp("", "incus_metadata_")
if err != nil {
_ = metaTarWriter.Close()
if rootfsTarWriter != nil {
_ = rootfsTarWriter.Close()
}
d.logger.Error("Failed exporting instance", ctxMap)
return nil, err
}
defer logger.WarnOnError(func() error { return os.RemoveAll(tempDir) }, "Failed to remove temporary directory")
data, err := yaml.Dump(&meta, yaml.WithV2Defaults())
if err != nil {
_ = metaTarWriter.Close()
if rootfsTarWriter != nil {
_ = rootfsTarWriter.Close()
}
d.logger.Error("Failed exporting instance", ctxMap)
return nil, err
}
fnam = filepath.Join(tempDir, "metadata.yaml")
err = os.WriteFile(fnam, data, 0o644)
if err != nil {
_ = metaTarWriter.Close()
if rootfsTarWriter != nil {
_ = rootfsTarWriter.Close()
}
d.logger.Error("Failed exporting instance", ctxMap)
return nil, err
}
// Add metadata.yaml to the tarball.
fi, err := os.Lstat(fnam)
if err != nil {
_ = metaTarWriter.Close()
if rootfsTarWriter != nil {
_ = rootfsTarWriter.Close()
}
d.logger.Error("Failed exporting instance", ctxMap)
return nil, err
}
tmpOffset := len(filepath.Dir(fnam)) + 1
err = metaTarWriter.WriteFile(fnam[tmpOffset:], fnam, fi, false)
if err != nil {
_ = metaTarWriter.Close()
if rootfsTarWriter != nil {
_ = rootfsTarWriter.Close()
}
d.logger.Debug("Error writing to tarfile", logger.Ctx{"err": err})
d.logger.Error("Failed exporting instance", ctxMap)
return nil, err
}
// If present, add config.json (OCI) to the tarball.
fnam = filepath.Join(d.Path(), "config.json")
if util.PathExists(fnam) {
fi, err := os.Lstat(fnam)
if err != nil {
_ = metaTarWriter.Close()
if rootfsTarWriter != nil {
_ = rootfsTarWriter.Close()
}
d.logger.Error("Failed exporting instance", ctxMap)
return nil, err
}
tmpOffset := len(filepath.Dir(fnam)) + 1
err = metaTarWriter.WriteFile(fnam[tmpOffset:], fnam, fi, false)
if err != nil {
_ = metaTarWriter.Close()
if rootfsTarWriter != nil {
_ = rootfsTarWriter.Close()
}
d.logger.Debug("Error writing to tarfile", logger.Ctx{"err": err})
d.logger.Error("Failed exporting instance", ctxMap)
return nil, err
}
}
// Include all the rootfs files.
fnam = d.RootfsPath()
if rootfsWriter == nil {
err = filepath.Walk(fnam, writeToMetaTar)
if err != nil {
d.logger.Error("Failed exporting instance", ctxMap)
return nil, err
}
} else {
err = filepath.Walk(fnam, writeToRootfsTar)
if err != nil {
d.logger.Error("Failed exporting instance", ctxMap)
return nil, err
}
}
// Include all the templates.
fnam = d.TemplatesPath()
if util.PathExists(fnam) {
err = filepath.Walk(fnam, writeToMetaTar)
if err != nil {
d.logger.Error("Failed exporting instance", ctxMap)
return nil, err
}
}
err = metaTarWriter.Close()
if err != nil {
d.logger.Error("Failed exporting instance", ctxMap)
return nil, err
}
if rootfsTarWriter != nil {
err = rootfsTarWriter.Close()
if err != nil {
d.logger.Error("Failed exporting instance", ctxMap)
return nil, err
}
}
d.logger.Info("Exported instance", ctxMap)
return &meta, nil
}
func collectCRIULogFile(d instance.Instance, imagesDir string, function string, method string) error {
t := time.Now().Format(time.RFC3339)
newPath := filepath.Join(d.LogPath(), fmt.Sprintf("%s_%s_%s.log", function, method, t))
return internalUtil.FileCopy(filepath.Join(imagesDir, fmt.Sprintf("%s.log", method)), newPath)
}
func getCRIULogErrors(imagesDir string, method string) (string, error) {
f, err := os.Open(path.Join(imagesDir, fmt.Sprintf("%s.log", method)))
if err != nil {
return "", err
}
defer logger.WarnOnError(f.Close, "Failed to close file")
scanner := bufio.NewScanner(f)
ret := []string{}
for scanner.Scan() {
line := scanner.Text()
if strings.Contains(line, "Error") || strings.Contains(line, "Warn") {
ret = append(ret, scanner.Text())
}
}
return strings.Join(ret, "\n"), nil
}
// Check if CRIU supports pre-dumping and number of pre-dump iterations.
func (d *lxc) migrationSendCheckForPreDumpSupport() (bool, int) {
// Check if this architecture/kernel/criu combination supports pre-copy dirty memory tracking feature.
_, err := subprocess.RunCommand("criu", "check", "--feature", "mem_dirty_track")
if err != nil {
// CRIU says it does not know about dirty memory tracking.
// This means the rest of this function is irrelevant.
return false, 0
}
// CRIU says it can actually do pre-dump. Let's set it to true
// unless the user wants something else.
usePreDumps := true
// What does the configuration say about pre-copy
tmp := d.ExpandedConfig()["migration.incremental.memory"]
if tmp != "" {
usePreDumps = util.IsTrue(tmp)
}
var maxIterations int
// migration.incremental.memory.iterations is the value after which the
// container will be definitely migrated, even if the remaining number
// of memory pages is below the defined threshold.
tmp = d.ExpandedConfig()["migration.incremental.memory.iterations"]
if tmp != "" {
maxIterations, _ = strconv.Atoi(tmp)
} else {
// default to 10
maxIterations = 10
}
if maxIterations > 999 {
// the pre-dump directory is hardcoded to a string
// with maximal 3 digits. 999 pre-dumps makes no
// sense at all, but let's make sure the number
// is not higher than this.
maxIterations = 999
}
logger.Debugf("Using maximal %d iterations for pre-dumping", maxIterations)
return usePreDumps, maxIterations
}
func (d *lxc) migrationSendWriteActionScript(directory string, operation string, secret string, execPath string) error {
script := fmt.Sprintf(`#!/bin/sh -e
if [ "$CRTOOLS_SCRIPT_ACTION" = "post-dump" ]; then
%s migratedumpsuccess %s %s
fi
`, execPath, operation, secret)
f, err := os.Create(filepath.Join(directory, "action.sh"))
if err != nil {
return err
}
err = f.Chmod(0o500)
if err != nil {
return err
}
_, err = f.WriteString(script)
if err != nil {
return err
}
return f.Close()
}
// MigrateSend sends an instance to a target for migration.
func (d *lxc) MigrateSend(args instance.MigrateSendArgs) error {
d.logger.Debug("Migration send starting")
defer d.logger.Debug("Migration send stopped")
// Check for an existing operation.
// Setup a new operation.
op := operationlock.Get(d.Project().Name, d.Name())
if op != nil && op.ActionMatch(operationlock.ActionMigrate) {
return errors.New("The instance is already being migrated")
}
op, err := operationlock.CreateWaitGet(d.Project().Name, d.Name(), d.op, operationlock.ActionMigrate, nil, false, true)
if err != nil {
return err
}
// If not running, stop any forkfile instance.
if !d.IsRunning() {
d.stopForkfile(false)
}
// Wait for essential migration connections before negotiation.
connectionsCtx, cancel := context.WithTimeout(context.Background(), time.Second*10)
defer cancel()
filesystemConn, err := args.FilesystemConn(connectionsCtx)
if err != nil {
op.Done(err)
return err
}
var stateConn io.ReadWriteCloser
if args.Live {
stateConn, err = args.StateConn(connectionsCtx)
if err != nil {
op.Done(err)
return err
}
}
pool, err := storagePools.LoadByInstance(d.state, d)
if err != nil {
err := fmt.Errorf("Failed loading instance: %w", err)
op.Done(err)
return err
}
clusterMove := args.ClusterMoveSourceName != ""
storageMove := args.StoragePool != ""
// The refresh argument passed to MigrationTypes() is always set to false here.
// The migration source/sender doesn't need to care whether or not it's doing a refresh as the migration
// sink/receiver will know this, and adjust the migration types accordingly.
// The same applies for clusterMove and storageMove, which are set to the most optimized defaults.
poolMigrationTypes := pool.MigrationTypes(storagePools.InstanceContentType(d), false, args.Snapshots, true, false)
if len(poolMigrationTypes) == 0 {
err := errors.New("No source migration types available")
op.Done(err)
return err
}
// Convert the pool's migration type options to an offer header to target.
// Populate the Fs, ZfsFeatures and RsyncFeatures fields.
offerHeader := localMigration.TypesToHeader(poolMigrationTypes...)
// Offer to send index header.
indexHeaderVersion := localMigration.IndexHeaderVersion
offerHeader.IndexHeaderVersion = &indexHeaderVersion
// Add CRIU and predump info to source header.
maxDumpIterations := 0
if args.Live {
var offerUsePreDumps bool
offerUsePreDumps, maxDumpIterations = d.migrationSendCheckForPreDumpSupport()
offerHeader.Predump = proto.Bool(offerUsePreDumps)
offerHeader.Criu = migration.CRIUType_CRIU_RSYNC.Enum()
} else {
offerHeader.Predump = proto.Bool(false)
if d.IsRunning() {
// Indicate instance is running to target (can trigger MultiSync mode).
offerHeader.Criu = migration.CRIUType_NONE.Enum()
}
}
// Add idmap info to source header for containers.
idmapset, err := d.DiskIdmap()
if err != nil {
err := fmt.Errorf("Failed getting container disk idmap: %w", err)
op.Done(err)
return err
} else if idmapset != nil {
offerHeader.Idmap = make([]*migration.IDMapType, 0, len(idmapset.Entries))
for _, ctnIdmap := range idmapset.Entries {
idmapEntry := migration.IDMapType{
Isuid: proto.Bool(ctnIdmap.IsUID),
Isgid: proto.Bool(ctnIdmap.IsGID),
Hostid: proto.Int32(int32(ctnIdmap.HostID)),
Nsid: proto.Int32(int32(ctnIdmap.NSID)),
Maprange: proto.Int32(int32(ctnIdmap.MapRange)),
}
offerHeader.Idmap = append(offerHeader.Idmap, &idmapEntry)
}
}
srcConfig, err := pool.GenerateInstanceBackupConfig(d, args.Snapshots, true, d.op)
if err != nil {
err := fmt.Errorf("Failed generating instance migration config: %w", err)
op.Done(err)
return err
}
dependentVolumesOffer, err := storagePools.GenerateDependentVolumesOffer(d.state, srcConfig, d.Project().Name, args.Snapshots, args.Devices, args.ClusterMoveSourceName != "")
if err != nil {
err := fmt.Errorf("Failed generating instance depending volumes offer: %w", err)
op.Done(err)
return err
}
offerHeader.DependentVolumes = dependentVolumesOffer
// If we are copying snapshots, retrieve a list of snapshots from source volume.
if args.Snapshots {
offerHeader.SnapshotNames = make([]string, 0, len(srcConfig.Snapshots))
offerHeader.Snapshots = make([]*migration.Snapshot, 0, len(srcConfig.Snapshots))
for i := range srcConfig.Snapshots {
offerHeader.SnapshotNames = append(offerHeader.SnapshotNames, srcConfig.Snapshots[i].Name)
offerHeader.Snapshots = append(offerHeader.Snapshots, instance.SnapshotToProtobuf(srcConfig.Snapshots[i]))
}
}
// Send offer to target.
d.logger.Debug("Sending migration offer to target")
err = args.ControlSend(offerHeader)
if err != nil {
err := fmt.Errorf("Failed sending migration offer: %w", err)
op.Done(err)
return err
}
// Receive response from target.
d.logger.Debug("Waiting for migration offer response from target")
respHeader := &migration.MigrationHeader{}
err = args.ControlReceive(respHeader, true)
if err != nil {
err := fmt.Errorf("Failed receiving migration offer response: %w", err)
op.Done(err)
return err
}
d.logger.Debug("Got migration offer response from target")
// Negotiated migration types.
migrationTypes, err := localMigration.MatchTypes(respHeader, migration.MigrationFSType_RSYNC, poolMigrationTypes)
if err != nil {
err := fmt.Errorf("Failed to negotiate migration type: %w", err)
op.Done(err)
return err
}
volumesWithTypes, err := storagePools.DependentVolumesMatchMigrationType(d.state, respHeader.DependentVolumes, args.Snapshots, nil, true)
if err != nil {
err := fmt.Errorf("Failed to negotiate migration types for dependent volumes: %w", err)
op.Done(err)
return err
}
d.logger.Debug("Generate dependent volumes args")
dependentVolumes := []localMigration.DependentVolumeArgs{}
for _, volWithType := range volumesWithTypes {
dependentVolumes = append(dependentVolumes, localMigration.ProtobufToDependentVolume(volWithType.Volume, volWithType.VolumeTypes[0], nil))
}
volSourceArgs := &localMigration.VolumeSourceArgs{
IndexHeaderVersion: respHeader.GetIndexHeaderVersion(), // Enable index header frame if supported.
Name: d.Name(),
MigrationType: migrationTypes[0],
Snapshots: offerHeader.SnapshotNames,
TrackProgress: true,
Refresh: respHeader.GetRefresh(),
AllowInconsistent: args.AllowInconsistent,
VolumeOnly: !args.Snapshots,
Info: &localMigration.Info{Config: srcConfig},
ClusterMove: clusterMove,
StorageMove: storageMove,
DependentVolumes: dependentVolumes,
}
// Only send the snapshots that the target requests when refreshing.
if respHeader.GetRefresh() {
volSourceArgs.Snapshots = respHeader.GetSnapshotNames()
allSnapshots := volSourceArgs.Info.Config.VolumeSnapshots
// Ensure that only the requested snapshots are included in the migration index header.
volSourceArgs.Info.Config.VolumeSnapshots = make([]*api.StorageVolumeSnapshot, 0, len(volSourceArgs.Snapshots))
for i := range allSnapshots {
if slices.Contains(volSourceArgs.Snapshots, allSnapshots[i].Name) {
volSourceArgs.Info.Config.VolumeSnapshots = append(volSourceArgs.Info.Config.VolumeSnapshots, allSnapshots[i])
}
}
}
// If s.live is true or Criu is set to CRIUType_NONE rather than nil, it indicates that the source instance
// is running, and if we are doing a non-optimized transfer (i.e using rsync or raw block transfer) then we
// should do a two stage transfer to minimize downtime.
instanceRunning := args.Live || (respHeader.Criu != nil && *respHeader.Criu == migration.CRIUType_NONE)
nonOptimizedMigration := volSourceArgs.MigrationType.FSType == migration.MigrationFSType_RSYNC || volSourceArgs.MigrationType.FSType == migration.MigrationFSType_BLOCK_AND_RSYNC
if instanceRunning && nonOptimizedMigration {
// Indicate this info to the storage driver so that it can alter its behaviour if needed.
volSourceArgs.MultiSync = true
}
g, ctx := errgroup.WithContext(context.Background())
// Start control connection monitor.
g.Go(func() error {
d.logger.Debug("Migrate send control monitor started")
defer d.logger.Debug("Migrate send control monitor finished")
controlResult := make(chan error, 1) // Buffered to allow go routine to end if no readers.
// This will read the result message from the target side and detect disconnections.
go func() {
resp := migration.MigrationControl{}
err := args.ControlReceive(&resp, false)
if err != nil {
err = fmt.Errorf("Error reading migration control target: %w", err)
} else if !resp.GetSuccess() {
err = fmt.Errorf("Error from migration control target: %s", resp.GetMessage())
}
controlResult <- err
}()
// End as soon as we get control message/disconnection from the target side or a local error.
select {
case <-ctx.Done():
err = ctx.Err()
case err = <-controlResult:
}
return err
})
// Start error monitoring routine, this will detect when an error is returned from the other routines,
// and if that happens it will disconnect the migration connections which will trigger the other routines
// to finish.
go func() {
<-ctx.Done()
args.Disconnect()
}()
restoreSuccess := make(chan bool, 1)
defer close(restoreSuccess)
// Don't defer close this one as its needed potentially after this function has ended.
dumpSuccess := make(chan error, 1)
g.Go(func() error {
d.logger.Debug("Migrate send transfer started")
defer d.logger.Debug("Migrate send transfer finished")
var err error
d.logger.Debug("Starting storage migration phase")
err = pool.MigrateInstance(d, filesystemConn, volSourceArgs, d.op)
if err != nil {
return err
}
d.logger.Debug("Finished storage migration phase")
if args.Live {
d.logger.Debug("Starting live migration phase")
// Setup rsync options (used for CRIU state transfers).
rsyncBwlimit := pool.Driver().Config()["rsync.bwlimit"]
rsyncFeatures := respHeader.GetRsyncFeaturesSlice()
if !slices.Contains(rsyncFeatures, "bidirectional") {
// If no bi-directional support, assume 3.7 level.
// NOTE: Do NOT extend this list of arguments.
rsyncFeatures = []string{"xattrs", "delete", "compress"}
}
if respHeader.Criu == nil {
return errors.New("Got no CRIU socket type for live migration")
} else if *respHeader.Criu != migration.CRIUType_CRIU_RSYNC {
return fmt.Errorf("Formats other than criu rsync not understood (%q)", respHeader.Criu)
}
checkpointDir, err := os.MkdirTemp("", "incus_checkpoint_")
if err != nil {
return err
}
if liblxc.RuntimeLiblxcVersionAtLeast(liblxc.Version(), 2, 0, 4) {
// What happens below is slightly convoluted. Due to various complications
// with networking, there's no easy way for criu to exit and leave the
// container in a frozen state for us to somehow resume later.
// Instead, we use what criu calls an "action-script", which is basically a
// callback that lets us know when the dump is done. (Unfortunately, we
// can't pass arguments, just an executable path, so we write a custom
// action script with the real command we want to run.)
// This script then blocks until the migration operation either finishes
// successfully or fails, and exits 1 or 0, which causes criu to either
// leave the container running or kill it as we asked.
dumpDone := make(chan bool, 1)
actionScriptOpSecret, err := internalUtil.RandomHexString(32)
if err != nil {
_ = os.RemoveAll(checkpointDir)
return err
}
actionScriptOp, err := operations.OperationCreate(
d.state,
d.Project().Name,
operations.OperationClassWebsocket,
operationtype.InstanceLiveMigrate,
nil,
nil,
func(op *operations.Operation) error {
result := <-restoreSuccess
if !result {
return errors.New("restore failed, failing CRIU")
}
return nil
},
nil,
func(op *operations.Operation, r *http.Request, w http.ResponseWriter) error {
secret := r.FormValue("secret")
if secret == "" {
return errors.New("Missing action script secret")
}
if secret != actionScriptOpSecret {
return os.ErrPermission
}
c, err := ws.Upgrader.Upgrade(w, r, nil)
if err != nil {
return err
}
dumpDone <- true
closeMsg := websocket.FormatCloseMessage(websocket.CloseNormalClosure, "")
return c.WriteMessage(websocket.CloseMessage, closeMsg)
},
nil,
)
if err != nil {
_ = os.RemoveAll(checkpointDir)
return err
}
err = d.migrationSendWriteActionScript(checkpointDir, actionScriptOp.URL(), actionScriptOpSecret, d.state.OS.ExecPath)
if err != nil {
_ = os.RemoveAll(checkpointDir)
return err
}
preDumpCounter := 0
preDumpDir := ""
// Check if the other side knows about pre-dumping and the associated
// rsync protocol.
if respHeader.GetPredump() {
d.logger.Debug("The other side does support pre-copy")
final := false
for !final {
preDumpCounter++
if preDumpCounter < maxDumpIterations {
final = false
} else {
final = true
}
dumpDir := fmt.Sprintf("%03d", preDumpCounter)
loopArgs := preDumpLoopArgs{
stateConn: stateConn,
checkpointDir: checkpointDir,
bwlimit: rsyncBwlimit,
preDumpDir: preDumpDir,
dumpDir: dumpDir,
final: final,
rsyncFeatures: rsyncFeatures,
}
final, err = d.migrateSendPreDumpLoop(&loopArgs)
if err != nil {
_ = os.RemoveAll(checkpointDir)
return err
}
preDumpDir = fmt.Sprintf("%03d", preDumpCounter)
preDumpCounter++
}
} else {
d.logger.Debug("The other side does not support pre-copy")
}
err = actionScriptOp.Start()
if err != nil {
_ = os.RemoveAll(checkpointDir)
return err
}
go func() {
d.logger.Debug("Final CRIU dump started")
defer d.logger.Debug("Final CRIU dump stopped")
criuMigrationArgs := instance.CriuMigrationArgs{
Cmd: liblxc.MIGRATE_DUMP,
Stop: true,
ActionScript: true,
PreDumpDir: preDumpDir,
DumpDir: "final",
StateDir: checkpointDir,
Function: "migration",
}
// Do the final CRIU dump. This is needs no special handling if
// pre-dumps are used or not.
dumpSuccess <- d.migrate(&criuMigrationArgs)
_ = os.RemoveAll(checkpointDir)
}()
select {
// The checkpoint failed, let's just abort.
case err = <-dumpSuccess:
return err
// The dump finished, let's continue on to the restore.
case <-dumpDone:
d.logger.Debug("Dump finished, continuing with restore...")
}
} else {
d.logger.Debug("The version of liblxc is older than 2.0.4 and the live migration will probably fail")
defer logger.WarnOnError(func() error { return os.RemoveAll(checkpointDir) }, "Failed to remove checkpoint directory")
criuMigrationArgs := instance.CriuMigrationArgs{
Cmd: liblxc.MIGRATE_DUMP,
StateDir: checkpointDir,
Function: "migration",
Stop: true,
ActionScript: false,
DumpDir: "final",
PreDumpDir: "",
}
err = d.migrate(&criuMigrationArgs)
if err != nil {
return err
}
}
// We do the transfer serially right now, but there's really no reason for us to;
// since we have separate websockets, we can do it in parallel if we wanted to.
// However assuming we're network bound, there's really no reason to do these in.
// parallel. In the future when we're using p.haul's protocol, it will make sense
// to do these in parallel.
ctName, _, _ := api.GetParentAndSnapshotName(d.Name())
err = rsync.Send(ctName, internalUtil.AddSlash(checkpointDir), stateConn, nil, rsyncFeatures, rsyncBwlimit, d.state.OS.ExecPath)
if err != nil {
return err
}
d.logger.Debug("Finished live migration phase")
}
// Perform final sync if in multi sync mode.
if volSourceArgs.MultiSync {
d.logger.Debug("Starting final storage migration phase")
// Indicate to the storage driver we are doing final sync and because of this don't send
// snapshots as they don't need to have a final sync as not being modified.
volSourceArgs.FinalSync = true
volSourceArgs.Snapshots = nil
volSourceArgs.Info.Config.VolumeSnapshots = nil
err = pool.MigrateInstance(d, filesystemConn, volSourceArgs, d.op)
if err != nil {
return err
}
d.logger.Debug("Finished final storage migration phase")
}
return nil
})
{
// Wait for routines to finish and collect first error.
err := g.Wait()
if args.Live {
restoreSuccess <- err == nil
if err == nil {
err := <-dumpSuccess
if err != nil {
d.logger.Error("Dump failed after successful restore", logger.Ctx{"err": err})
}
}
}
if err != nil {
op.Done(err)
return err
}
op.Done(nil)
d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceMigrated.Event(d, nil))
return nil
}
}
type preDumpLoopArgs struct {
stateConn io.ReadWriteCloser
checkpointDir string
bwlimit string
preDumpDir string
dumpDir string
final bool
rsyncFeatures []string
}
// migrateSendPreDumpLoop is the main logic behind the pre-copy migration.
// This function contains the actual pre-dump, the corresponding rsync transfer and it tells the outer loop to
// abort if the threshold of memory pages transferred by pre-dumping has been reached.
func (d *lxc) migrateSendPreDumpLoop(args *preDumpLoopArgs) (bool, error) {
// Do a CRIU pre-dump
criuMigrationArgs := instance.CriuMigrationArgs{
Cmd: liblxc.MIGRATE_PRE_DUMP,
Stop: false,
ActionScript: false,
PreDumpDir: args.preDumpDir,
DumpDir: args.dumpDir,
StateDir: args.checkpointDir,
Function: "migration",
}
d.logger.Debug("Doing another CRIU pre-dump", logger.Ctx{"preDumpDir": args.preDumpDir})
final := args.final
if d.Type() != instancetype.Container {
return false, errors.New("Instance is not container type")
}
err := d.migrate(&criuMigrationArgs)
if err != nil {
return final, fmt.Errorf("Failed sending instance: %w", err)
}
// Send the pre-dump.
ctName, _, _ := api.GetParentAndSnapshotName(d.Name())
err = rsync.Send(ctName, internalUtil.AddSlash(args.checkpointDir), args.stateConn, nil, args.rsyncFeatures, args.bwlimit, d.state.OS.ExecPath)
if err != nil {
return final, err
}
// The function readCriuStatsDump() reads the CRIU 'stats-dump' file
// in path and returns the pages_written, pages_skipped_parent, error.
readCriuStatsDump := func(statsPath string) (uint64, uint64, error) {
// Get dump statistics with crit
dumpStats, err := crit.GetDumpStats(statsPath)
if err != nil {
return 0, 0, fmt.Errorf("Failed to parse CRIU's 'stats-dump' file: %w", err)
}
return dumpStats.GetPagesWritten(), dumpStats.GetPagesSkippedParent(), nil
}
// Read the CRIU's 'stats-dump' file
dumpPath := internalUtil.AddSlash(args.checkpointDir)
dumpPath += internalUtil.AddSlash(args.dumpDir)
written, skippedParent, err := readCriuStatsDump(dumpPath)
if err != nil {
return final, err
}
totalPages := written + skippedParent
var percentageSkipped int
if totalPages > 0 {
percentageSkipped = int(100 - ((100 * written) / totalPages))
}
d.logger.Debug("CRIU pages", logger.Ctx{"pages": written, "skipped": skippedParent, "skippedPerc": percentageSkipped})
// threshold is the percentage of memory pages that needs
// to be pre-copied for the pre-copy migration to stop.
var threshold int
tmp := d.ExpandedConfig()["migration.incremental.memory.goal"]
if tmp != "" {
threshold, _ = strconv.Atoi(tmp)
} else {
// defaults to 70%
threshold = 70
}
if percentageSkipped > threshold {
d.logger.Debug("Memory pages skipped due to pre-copy is larger than threshold", logger.Ctx{"skippedPerc": percentageSkipped, "thresholdPerc": threshold})
d.logger.Debug("This was the last pre-dump; next dump is the final dump")
final = true
}
// If in pre-dump mode, the receiving side expects a message to know if this was the last pre-dump.
logger.Debug("Sending another CRIU pre-dump header")
syncMsg := migration.MigrationSync{
FinalPreDump: proto.Bool(final),
}
data, err := proto.Marshal(&syncMsg)
if err != nil {
return false, err
}
_, err = args.stateConn.Write(data)
if err != nil {
return final, err
}
d.logger.Debug("Sending another CRIU pre-dump header done")
return final, nil
}
func (d *lxc) resetContainerDiskIdmap(srcIdmap *idmap.Set) error {
dstIdmap, err := d.DiskIdmap()
if err != nil {
return err
}
if dstIdmap == nil {
dstIdmap = &idmap.Set{}
}
if !srcIdmap.Equals(dstIdmap) {
jsonIdmap, err := srcIdmap.ToJSON()
if err != nil {
return fmt.Errorf("Failed to encode ID map: %w", err)
}
d.logger.Debug("Setting new volatile.last_state.idmap from source instance", logger.Ctx{"sourceIdmap": srcIdmap})
err = d.VolatileSet(map[string]string{"volatile.last_state.idmap": jsonIdmap})
if err != nil {
return err
}
}
return nil
}
// MigrateReceive receives an instance being migrated from a source.
func (d *lxc) MigrateReceive(args instance.MigrateReceiveArgs) error {
d.logger.Debug("Migration receive starting")
defer d.logger.Debug("Migration receive stopped")
// Wait for essential migration connections before negotiation.
connectionsCtx, cancel := context.WithTimeout(context.Background(), time.Second*10)
defer cancel()
filesystemConn, err := args.FilesystemConn(connectionsCtx)
if err != nil {
return err
}
var stateConn io.ReadWriteCloser
if args.Live {
stateConn, err = args.StateConn(connectionsCtx)
if err != nil {
return err
}
}
// Receive offer from source.
d.logger.Debug("Waiting for migration offer from source")
offerHeader := &migration.MigrationHeader{}
err = args.ControlReceive(offerHeader, true)
if err != nil {
return fmt.Errorf("Failed receiving migration offer from source: %w", err)
}
criuType := migration.CRIUType_CRIU_RSYNC.Enum()
if offerHeader.Criu != nil && *offerHeader.Criu == migration.CRIUType_NONE {
criuType = migration.CRIUType_NONE.Enum()
} else {
if !args.Live {
criuType = nil
}
}
// When doing a cluster same-name move we cannot load the storage pool using the instance's volume DB
// record because it may be associated to the wrong cluster member. Instead we ascertain the pool to load
// using the instance's root disk device.
if args.ClusterMoveSourceName == d.name {
_, rootDiskDevice, err := d.getRootDiskDevice()
if err != nil {
return fmt.Errorf("Failed getting root disk: %w", err)
}
if rootDiskDevice["pool"] == "" {
return errors.New("The instance's root device is missing the pool property")
}
// Initialize the storage pool cache.
d.storagePool, err = storagePools.LoadByName(d.state, rootDiskDevice["pool"])
if err != nil {
return fmt.Errorf("Failed loading storage pool: %w", err)
}
}
pool, err := storagePools.LoadByInstance(d.state, d)
if err != nil {
return err
}
// The source will never set Refresh in the offer header.
// However, to determine the correct migration type Refresh needs to be set.
offerHeader.Refresh = &args.Refresh
clusterMove := args.ClusterMoveSourceName != ""
storageMove := args.StoragePool != ""
// Extract the source's migration type and then match it against our pool's supported types and features.
// If a match is found the combined features list will be sent back to requester.
contentType := storagePools.InstanceContentType(d)
respTypes, err := localMigration.MatchTypes(offerHeader, storagePools.FallbackMigrationType(contentType), pool.MigrationTypes(contentType, args.Refresh, args.Snapshots, clusterMove, storageMove))
if err != nil {
return err
}
// The migration header to be sent back to source with our target options.
// Convert response type to response header and copy snapshot info into it.
respHeader := localMigration.TypesToHeader(respTypes...)
// Respond with our maximum supported header version if the requested version is higher than ours.
// Otherwise just return the requested header version to the source.
indexHeaderVersion := min(offerHeader.GetIndexHeaderVersion(), localMigration.IndexHeaderVersion)
respHeader.IndexHeaderVersion = &indexHeaderVersion
respHeader.SnapshotNames = offerHeader.SnapshotNames
respHeader.Snapshots = offerHeader.Snapshots
respHeader.Refresh = &args.Refresh
// Add CRIU info to response.
respHeader.Criu = criuType
localDevices := d.LocalDevices().CloneNative()
volumesWithTypes, err := storagePools.DependentVolumesMatchMigrationType(d.state, offerHeader.DependentVolumes, args.Snapshots, localDevices, false)
if err != nil {
return fmt.Errorf("Failed to negotiate migration types for dependent volumes: %w", err)
}
dependentVolumes := []localMigration.DependentVolumeArgs{}
for _, volWithType := range volumesWithTypes {
respHeader.DependentVolumes = append(respHeader.DependentVolumes, volWithType.Volume)
vol := localMigration.ProtobufToDependentVolume(volWithType.Volume, volWithType.VolumeTypes[0], localDevices[*volWithType.Volume.DeviceName])
dependentVolumes = append(dependentVolumes, vol)
}
if args.Refresh {
// Get the remote snapshots on the source.
sourceSnapshots := offerHeader.GetSnapshots()
sourceSnapshotComparable := make([]storagePools.ComparableSnapshot, 0, len(sourceSnapshots))
for _, sourceSnap := range sourceSnapshots {
sourceSnapshotComparable = append(sourceSnapshotComparable, storagePools.ComparableSnapshot{
Name: sourceSnap.GetName(),
CreationDate: time.Unix(sourceSnap.GetCreationDate(), 0),
})
}
// Get existing snapshots on the local target.
targetSnapshots, err := d.Snapshots()
if err != nil {
return err
}
targetSnapshotsComparable := make([]storagePools.ComparableSnapshot, 0, len(targetSnapshots))
for _, targetSnap := range targetSnapshots {
_, targetSnapName, _ := api.GetParentAndSnapshotName(targetSnap.Name())
targetSnapshotsComparable = append(targetSnapshotsComparable, storagePools.ComparableSnapshot{
Name: targetSnapName,
CreationDate: targetSnap.CreationDate(),
})
}
// Compare the two sets.
syncSourceSnapshotIndexes, deleteTargetSnapshotIndexes := storagePools.CompareSnapshots(sourceSnapshotComparable, targetSnapshotsComparable, args.RefreshExcludeOlder)
// Delete the extra local snapshots first.
for _, deleteTargetSnapshotIndex := range deleteTargetSnapshotIndexes {
err := targetSnapshots[deleteTargetSnapshotIndex].Delete(true, true)
if err != nil {
return err
}
}
// Only request to send the snapshots that need updating.
syncSnapshotNames := make([]string, 0, len(syncSourceSnapshotIndexes))
syncSnapshots := make([]*migration.Snapshot, 0, len(syncSourceSnapshotIndexes))
for _, syncSourceSnapshotIndex := range syncSourceSnapshotIndexes {
syncSnapshotNames = append(syncSnapshotNames, sourceSnapshots[syncSourceSnapshotIndex].GetName())
syncSnapshots = append(syncSnapshots, sourceSnapshots[syncSourceSnapshotIndex])
}
respHeader.Snapshots = syncSnapshots
respHeader.SnapshotNames = syncSnapshotNames
offerHeader.Snapshots = syncSnapshots
offerHeader.SnapshotNames = syncSnapshotNames
}
if offerHeader.GetPredump() {
// If the other side wants pre-dump and if this side supports it, let's use it.
respHeader.Predump = proto.Bool(true)
} else {
respHeader.Predump = proto.Bool(false)
}
// Get rsync options from sender, these are passed into mySink function as part of
// MigrationSinkArgs below.
rsyncFeatures := respHeader.GetRsyncFeaturesSlice()
// Send response to source.
d.logger.Debug("Sending migration response to source")
err = args.ControlSend(respHeader)
if err != nil {
return fmt.Errorf("Failed sending migration response to source: %w", err)
}
d.logger.Debug("Sent migration response to source")
srcIdmap := &idmap.Set{}
for _, idmapSet := range offerHeader.Idmap {
e := idmap.Entry{
IsUID: *idmapSet.Isuid,
IsGID: *idmapSet.Isgid,
NSID: int64(*idmapSet.Nsid),
HostID: int64(*idmapSet.Hostid),
MapRange: int64(*idmapSet.Maprange),
}
srcIdmap.Entries = append(srcIdmap.Entries, e)
}
reverter := revert.New()
defer reverter.Fail()
g, ctx := errgroup.WithContext(context.Background())
// Start control connection monitor.
g.Go(func() error {
d.logger.Debug("Migrate receive control monitor started")
defer d.logger.Debug("Migrate receive control monitor finished")
controlResult := make(chan error, 1) // Buffered to allow go routine to end if no readers.
// This will read the result message from the source side and detect disconnections.
go func() {
resp := migration.MigrationControl{}
err := args.ControlReceive(&resp, false)
if err != nil {
err = fmt.Errorf("Error reading migration control source: %w", err)
} else if !resp.GetSuccess() {
err = fmt.Errorf("Error from migration control source: %s", resp.GetMessage())
}
controlResult <- err
}()
// End as soon as we get control message/disconnection from the source side or a local error.
select {
case <-ctx.Done():
err = ctx.Err()
case err = <-controlResult:
}
return err
})
// Start error monitoring routine, this will detect when an error is returned from the other routines,
// and if that happens it will disconnect the migration connections which will trigger the other routines
// to finish.
go func() {
<-ctx.Done()
args.Disconnect()
}()
// Start filesystem transfer routine and initialize a channel that is closed when the routine finishes.
fsTransferDone := make(chan struct{})
g.Go(func() error {
defer close(fsTransferDone)
d.logger.Debug("Migrate receive filesystem transfer started")
defer d.logger.Debug("Migrate receive filesystem transfer finished")
var err error
// We do the fs receive in parallel so we don't have to reason about when to receive
// what. The sending side is smart enough to send the filesystem bits that it can
// before it seizes the container to start checkpointing, so the total transfer time
// will be minimized even if we're dumb here.
snapshots := []*migration.Snapshot{}
// Legacy: we only sent the snapshot names, so we just copy the container's
// config over, same as we used to do.
if len(offerHeader.SnapshotNames) != len(offerHeader.Snapshots) {
// Convert the instance to an api.InstanceSnapshot.
profileNames := make([]string, 0, len(d.Profiles()))
for _, p := range d.Profiles() {
profileNames = append(profileNames, p.Name)
}
architectureName, _ := osarch.ArchitectureName(d.Architecture())
apiInstSnap := &api.InstanceSnapshot{
InstanceSnapshotPut: api.InstanceSnapshotPut{
ExpiresAt: time.Time{},
},
Architecture: architectureName,
CreatedAt: d.CreationDate(),
LastUsedAt: d.LastUsedDate(),
Config: d.LocalConfig(),
Description: d.Description(),
Devices: d.LocalDevices().CloneNative(),
Ephemeral: d.IsEphemeral(),
Stateful: d.IsStateful(),
Profiles: profileNames,
}
for _, name := range offerHeader.SnapshotNames {
base := instance.SnapshotToProtobuf(apiInstSnap)
base.Name = &name
snapshots = append(snapshots, base)
}
} else {
snapshots = offerHeader.Snapshots
}
// Default to not expecting to receive the final rootfs sync.
sendFinalFsDelta := false
// If we are doing a stateful live transfer or the CRIU type indicates we
// are doing a stateless transfer with a running instance then we should
// expect the source to send us a final rootfs sync.
if args.Live {
sendFinalFsDelta = true
} else if criuType != nil && *criuType == migration.CRIUType_NONE {
sendFinalFsDelta = true
}
volTargetArgs := localMigration.VolumeTargetArgs{
IndexHeaderVersion: respHeader.GetIndexHeaderVersion(),
Name: d.Name(),
MigrationType: respTypes[0],
Refresh: args.Refresh, // Indicate to receiver volume should exist.
TrackProgress: true, // Use a progress tracker on receiver to get in-cluster progress information.
Live: sendFinalFsDelta, // Indicates we will get a final rootfs sync.
VolumeSize: offerHeader.GetVolumeSize(), // Block size setting override.
VolumeOnly: !args.Snapshots,
ClusterMoveSourceName: args.ClusterMoveSourceName,
StoragePool: args.StoragePool,
DependentVolumes: dependentVolumes,
}
// At this point we have already figured out the parent container's root
// disk device so we can simply retrieve it from the expanded devices.
parentStoragePool := ""
parentExpandedDevices := d.ExpandedDevices()
parentLocalRootDiskDeviceKey, parentLocalRootDiskDevice, _ := internalInstance.GetRootDiskDevice(parentExpandedDevices.CloneNative())
if parentLocalRootDiskDeviceKey != "" {
parentStoragePool = parentLocalRootDiskDevice["pool"]
}
if parentStoragePool == "" {
return errors.New("Instance's root device is missing the pool property")
}
// A zero length Snapshots slice indicates volume only migration in
// VolumeTargetArgs. So if VolumeOnly was requested, do not populate them.
if args.Snapshots {
volTargetArgs.Snapshots = make([]*migration.Snapshot, 0, len(snapshots))
for _, snap := range snapshots {
volTargetArgs.Snapshots = append(volTargetArgs.Snapshots, &migration.Snapshot{Name: snap.Name})
// Only create snapshot instance DB records if not doing a cluster same-name move.
// As otherwise the DB records will already exist.
if args.ClusterMoveSourceName != d.name {
snapArgs, err := instance.SnapshotProtobufToInstanceArgs(d.state, d, snap)
if err != nil {
return err
}
// Ensure that snapshot and parent container have the same
// storage pool in their local root disk device. If the root
// disk device for the snapshot comes from a profile on the
// new instance as well we don't need to do anything.
if snapArgs.Devices != nil {
snapLocalRootDiskDeviceKey, _, _ := internalInstance.GetRootDiskDevice(snapArgs.Devices.CloneNative())
if snapLocalRootDiskDeviceKey != "" {
snapArgs.Devices[snapLocalRootDiskDeviceKey]["pool"] = parentStoragePool
}
}
// Create the snapshot instance.
_, snapInstOp, cleanup, err := instance.CreateInternal(d.state, *snapArgs, d.op, true, false, false)
if err != nil {
return fmt.Errorf("Failed creating instance snapshot record %q: %w", snapArgs.Name, err)
}
reverter.Add(cleanup)
defer snapInstOp.Done(err)
}
}
}
err = pool.CreateInstanceFromMigration(d, filesystemConn, volTargetArgs, d.op)
if err != nil {
return fmt.Errorf("Failed creating instance on target: %w", err)
}
isRemoteClusterMove := clusterMove && pool.Driver().Info().Remote
// Only delete all instance volumes on error if the pool volume creation has succeeded to
// avoid deleting an existing conflicting volume.
if !volTargetArgs.Refresh && !isRemoteClusterMove {
reverter.Add(func() {
snapshots, _ := d.Snapshots()
snapshotCount := len(snapshots)
for k := range snapshots {
// Delete the snapshots in reverse order.
k = snapshotCount - 1 - k
_ = pool.DeleteInstanceSnapshot(snapshots[k], nil)
}
_ = pool.DeleteInstance(d, nil)
})
}
// For containers, the fs map of the source is sent as part of the migration
// stream, then at the end we need to record that map as last_state so that
// shifting can happen on startup if needed.
err = d.resetContainerDiskIdmap(srcIdmap)
if err != nil {
return err
}
if args.ClusterMoveSourceName != d.name {
err = d.DeferTemplateApply(instance.TemplateTriggerCopy)
if err != nil {
return err
}
}
return nil
})
// Start live state transfer routine (if required) and initialize a channel that is closed when the
// routine finishes. It is never closed if the routine is not started.
stateTransferDone := make(chan struct{})
if args.Live {
g.Go(func() error {
d.logger.Debug("Migrate receive state transfer started")
defer d.logger.Debug("Migrate receive state transfer finished")
defer close(stateTransferDone)
imagesDir, err := os.MkdirTemp("", "incus_restore_")
if err != nil {
return err
}
defer logger.WarnOnError(func() error { return os.RemoveAll(imagesDir) }, "Failed to remove images directory")
sync := &migration.MigrationSync{
FinalPreDump: proto.Bool(false),
}
if respHeader.GetPredump() {
for !sync.GetFinalPreDump() {
d.logger.Debug("Waiting to receive pre-dump rsync")
// Transfer a CRIU pre-dump.
err = rsync.Recv(internalUtil.AddSlash(imagesDir), stateConn, nil, rsyncFeatures)
if err != nil {
return fmt.Errorf("Failed receiving pre-dump rsync: %w", err)
}
d.logger.Debug("Done receiving pre-dump rsync")
d.logger.Debug("Waiting to receive pre-dump header")
// We can't use io.ReadAll here because sender doesn't call Close() to
// send the frame end indicator after writing the pre-dump header.
// So define a small buffer sufficient to fit migration.MigrationSync and
// then read what we have into it.
buf := make([]byte, 128)
n, err := stateConn.Read(buf)
if err != nil {
return fmt.Errorf("Failed receiving pre-dump header: %w", err)
}
err = proto.Unmarshal(buf[:n], sync)
if err != nil {
return fmt.Errorf("Failed unmarshalling pre-dump header: %w (%v)", err, string(buf))
}
d.logger.Debug("Done receiving pre-dump header")
}
}
// Final CRIU dump.
d.logger.Debug("About to receive final dump rsync")
err = rsync.Recv(internalUtil.AddSlash(imagesDir), stateConn, nil, rsyncFeatures)
if err != nil {
return fmt.Errorf("Failed receiving final dump rsync: %w", err)
}
d.logger.Debug("Done receiving final dump rsync")
// Wait until filesystem transfer is done before starting final state sync and restore.
<-fsTransferDone
// But only proceed if no errors have occurred thus far.
err = ctx.Err()
if err != nil {
return err
}
criuMigrationArgs := instance.CriuMigrationArgs{
Cmd: liblxc.MIGRATE_RESTORE,
StateDir: imagesDir,
Function: "migration",
Stop: false,
ActionScript: false,
DumpDir: "final",
PreDumpDir: "",
}
// Currently we only do a single CRIU pre-dump so we can hardcode "final"
// here since we know that "final" is the folder for CRIU's final dump.
err = d.migrate(&criuMigrationArgs)
if err != nil {
d.cleanupFailedMigrationRestore()
return err
}
return nil
})
}
{
// Wait until the filesystem transfer and state transfer routines have finished.
<-fsTransferDone
if args.Live {
<-stateTransferDone
}
// If context is cancelled by this stage, then an error has occurred.
// Wait for all routines to finish and collect the first error that occurred.
if ctx.Err() != nil {
err := g.Wait()
// Send failure response to source.
msg := migration.MigrationControl{
Success: proto.Bool(err == nil),
}
if err != nil {
msg.Message = proto.String(err.Error())
}
d.logger.Debug("Sending migration failure response to source", logger.Ctx{"err": err})
sendErr := args.ControlSend(&msg)
if sendErr != nil {
d.logger.Warn("Failed sending migration failure to source", logger.Ctx{"err": sendErr})
}
return err
}
// Send success response to source to control as nothing has gone wrong so far.
msg := migration.MigrationControl{
Success: proto.Bool(true),
}
d.logger.Debug("Sending migration success response to source", logger.Ctx{"success": msg.GetSuccess()})
err := args.ControlSend(&msg)
if err != nil {
d.logger.Warn("Failed sending migration success to source", logger.Ctx{"err": err})
return fmt.Errorf("Failed sending migration success to source: %w", err)
}
// Wait for all routines to finish (in this case it will be the control monitor) but do
// not collect the error, as it will just be a disconnect error from the source.
_ = g.Wait()
reverter.Success()
return nil
}
}
// Migrate migrates the instance to another node.
func (d *lxc) migrate(args *instance.CriuMigrationArgs) error {
ctxMap := logger.Ctx{
"created": d.creationDate,
"ephemeral": d.ephemeral,
"used": d.lastUsedDate,
"statedir": args.StateDir,
"actionscript": args.ActionScript,
"predumpdir": args.PreDumpDir,
"features": args.Features,
"stop": args.Stop,
}
_, err := exec.LookPath("criu")
if err != nil {
return localMigration.ErrNoLiveMigration
}
d.logger.Info("Migrating container", ctxMap)
prettyCmd := ""
switch args.Cmd {
case liblxc.MIGRATE_PRE_DUMP:
prettyCmd = "pre-dump"
case liblxc.MIGRATE_DUMP:
prettyCmd = "dump"
case liblxc.MIGRATE_RESTORE:
prettyCmd = "restore"
default:
prettyCmd = "unknown"
d.logger.Warn("Unknown migrate call", logger.Ctx{"cmd": args.Cmd})
}
pool, err := d.getStoragePool()
if err != nil {
return err
}
preservesInodes := pool.Driver().Info().PreservesInodes
/* This feature was only added in 2.0.1, let's not ask for it
* before then or migrations will fail.
*/
if !liblxc.RuntimeLiblxcVersionAtLeast(liblxc.Version(), 2, 0, 1) {
preservesInodes = false
}
finalStateDir := args.StateDir
var migrateErr error
/* For restore, we need an extra fork so that we daemonize the monitor
* instead of having it be a child. So let's hijack the command
* here and do the extra fork.
*/
if args.Cmd == liblxc.MIGRATE_RESTORE {
// Check that we're not already running.
if d.IsRunning() {
return errors.New("The container is already running")
}
// Run the shared start code.
configPath, postStartHooks, err := d.startCommon()
if err != nil {
if args.Op != nil {
args.Op.Done(err)
}
return err
}
/*
* For unprivileged containers we need to shift the
* perms on the images images so that they can be
* opened by the process after it is in its user
* namespace.
*/
idmapset, err := d.CurrentIdmap()
if err != nil {
return err
}
if idmapset != nil {
storageType, err := d.getStorageType()
if err != nil {
return fmt.Errorf("Storage type: %w", err)
}
switch storageType {
case "zfs":
err = idmapset.ShiftPath(args.StateDir, storageDrivers.ShiftZFSSkipper)
case "btrfs":
err = storageDrivers.ShiftBtrfsRootfs(args.StateDir, idmapset)
default:
err = idmapset.ShiftPath(args.StateDir, nil)
}
if err != nil {
return err
}
}
if args.DumpDir != "" {
finalStateDir = fmt.Sprintf("%s/%s", args.StateDir, args.DumpDir)
}
_, migrateErr = subprocess.RunCommand(
d.state.OS.ExecPath,
"forkmigrate",
project.Instance(d.Project().Name, d.Name()),
d.state.OS.LxcPath,
configPath,
finalStateDir,
fmt.Sprintf("%v", preservesInodes),
)
if migrateErr == nil {
// Run any post start hooks.
err = d.runHooks(postStartHooks)
if err != nil {
if args.Op != nil {
args.Op.Done(err) // Must come before Stop() otherwise stop will not proceed.
}
// Attempt to stop container.
_ = d.Stop(false)
return err
}
}
} else {
// Load the go-lxc struct
var cc *liblxc.Container
if d.expandedConfig["raw.lxc"] != "" {
cc, err = d.initLXC(true)
if err != nil {
return err
}
err = d.loadRawLXCConfig(cc)
if err != nil {
return err
}
} else {
cc, err = d.initLXC(false)
if err != nil {
return err
}
}
script := ""
if args.ActionScript {
script = filepath.Join(args.StateDir, "action.sh")
}
if args.DumpDir != "" {
finalStateDir = fmt.Sprintf("%s/%s", args.StateDir, args.DumpDir)
}
// TODO: make this configurable? Ultimately I think we don't
// want to do that; what we really want to do is have "modes"
// of criu operation where one is "make this succeed" and the
// other is "make this fast". Anyway, for now, let's choose a
// really big size so it almost always succeeds, even if it is
// slow.
ghostLimit := uint64(256 * 1024 * 1024)
opts := liblxc.MigrateOptions{
Stop: args.Stop,
Directory: finalStateDir,
Verbose: true,
PreservesInodes: preservesInodes,
ActionScript: script,
GhostLimit: ghostLimit,
}
if args.PreDumpDir != "" {
opts.PredumpDir = fmt.Sprintf("../%s", args.PreDumpDir)
}
if !d.IsRunning() {
// otherwise the migration will needlessly fail
args.Stop = false
}
migrateErr = cc.Migrate(args.Cmd, opts)
}
collectErr := collectCRIULogFile(d, finalStateDir, args.Function, prettyCmd)
if collectErr != nil {
d.logger.Error("Error collecting checkpoint log file", logger.Ctx{"err": collectErr})
}
if migrateErr != nil {
log, err2 := getCRIULogErrors(finalStateDir, prettyCmd)
if err2 == nil {
d.logger.Warn("Failed migrating container", ctxMap)
migrateErr = fmt.Errorf("%s %s failed\n%s", args.Function, prettyCmd, log)
}
return migrateErr
}
d.logger.Info("Migrated container", ctxMap)
return nil
}
func (d *lxc) templateApplyNow(trigger instance.TemplateTrigger) error {
// If there's no metadata, just return
fname := filepath.Join(d.Path(), "metadata.yaml")
if !util.PathExists(fname) {
return nil
}
// Parse the metadata
content, err := os.ReadFile(fname)
if err != nil {
return fmt.Errorf("Failed to read metadata: %w", err)
}
metadata := &api.ImageMetadata{}
err = yaml.Load(content, &metadata)
if err != nil {
return fmt.Errorf("Could not parse %s: %w", fname, err)
}
// Find rootUID and rootGID
idmapset, err := d.DiskIdmap()
if err != nil {
return fmt.Errorf("Failed to set ID map: %w", err)
}
rootUID := int64(0)
rootGID := int64(0)
// Get the right uid and gid for the container
if idmapset != nil {
rootUID, rootGID = idmapset.ShiftIntoNS(0, 0)
}
// Figure out the container architecture
arch, err := osarch.ArchitectureName(d.architecture)
if err != nil {
arch, err = osarch.ArchitectureName(d.state.OS.Architectures[0])
if err != nil {
return fmt.Errorf("Failed to detect system architecture: %w", err)
}
}
// Generate the container metadata
containerMeta := make(map[string]string)
containerMeta["name"] = d.name
containerMeta["type"] = "container"
containerMeta["architecture"] = arch
if d.ephemeral {
containerMeta["ephemeral"] = "true"
} else {
containerMeta["ephemeral"] = "false"
}
if d.IsPrivileged() {
containerMeta["privileged"] = "true"
} else {
containerMeta["privileged"] = "false"
}
// Open the container's rootfs as an os.Root. All template
// operations will then be relative to this root, avoiding issues with
// template files or paths within those templates attempting to access the
// fhost rootfs.
rootfs, err := os.OpenRoot(d.RootfsPath())
if err != nil {
return fmt.Errorf("Failed to open instance rootfs path: %w", err)
}
defer logger.WarnOnError(rootfs.Close, "Failed to close rootfs")
// Go through the templates.
for tplPath, tpl := range metadata.Templates {
err = func(tplPath string, tpl *api.ImageMetadataTemplate) error {
var w *os.File
// Check if the template should be applied now
found := slices.Contains(tpl.When, string(trigger))
if !found {
return nil
}
// Perform some early security checks on the template itself.
relPath := strings.TrimLeft(tplPath, "/")
if filepath.Base(tpl.Template) != tpl.Template {
return errors.New("Template path is attempting to read outside of template directory")
}
tplDirStat, err := os.Lstat(d.TemplatesPath())
if err != nil {
return fmt.Errorf("Couldn't access template directory: %w", err)
}
if !tplDirStat.IsDir() {
return errors.New("Template directory isn't a regular directory")
}
tplFileStat, err := os.Lstat(filepath.Join(d.TemplatesPath(), tpl.Template))
if err != nil {
return fmt.Errorf("Couldn't access template file: %w", err)
}
if tplFileStat.Mode()&os.ModeSymlink == os.ModeSymlink {
return errors.New("Template file is a symlink")
}
// Create the directory hierarchy, this has to be done
// somewhat manually as we not only need to create anything that's missing
// but also set the correct uid/gid.
relDir := path.Dir(relPath)
parent := ""
for _, part := range strings.Split(relDir, "/") {
if part == "" || part == "." {
continue
}
cur := path.Join(parent, part)
err = rootfs.Mkdir(cur, 0o755)
if err == nil {
// Fix ownership of any directory we just created.
err = rootfs.Chown(cur, int(rootUID), int(rootGID))
if err != nil {
return fmt.Errorf("Failed to set ownership on template directory: %w", err)
}
} else if !errors.Is(err, fs.ErrExist) {
return fmt.Errorf("Failed to create template directory: %w", err)
}
parent = cur
}
// Check whether the target file already exists.
_, err = rootfs.Stat(relPath)
existing := err == nil
if err != nil && !errors.Is(err, fs.ErrNotExist) {
return fmt.Errorf("Failed to check template file: %w", err)
}
if existing {
if tpl.CreateOnly {
return nil
}
// Open the existing file
w, err = rootfs.OpenFile(relPath, os.O_WRONLY|os.O_TRUNC, 0)
if err != nil {
return fmt.Errorf("Failed to create template file: %w", err)
}
} else {
// UID and GID
fileUID := int64(0)
fileGID := int64(0)
if tpl.UID != "" {
id, err := strconv.ParseInt(tpl.UID, 10, 64)
if err != nil {
return fmt.Errorf("Bad file UID %q for %q: %w", tpl.UID, tplPath, err)
}
fileUID = id
}
if tpl.GID != "" {
id, err := strconv.ParseInt(tpl.GID, 10, 64)
if err != nil {
return fmt.Errorf("Bad file GID %q for %q: %w", tpl.GID, tplPath, err)
}
fileGID = id
}
if idmapset != nil {
fileUID, fileGID = idmapset.ShiftIntoNS(fileUID, fileGID)
}
// Mode
fileMode := fs.FileMode(0o644)
if tpl.Mode != "" {
if len(tpl.Mode) == 3 {
tpl.Mode = fmt.Sprintf("0%s", tpl.Mode)
}
mode, err := strconv.ParseInt(tpl.Mode, 0, 0)
if err != nil {
return fmt.Errorf("Bad mode %q for %q: %w", tpl.Mode, tplPath, err)
}
fileMode = os.FileMode(mode) & os.ModePerm
}
// Create the file itself beneath the rootfs.
w, err = rootfs.OpenFile(relPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, fileMode.Perm())
if err != nil {
return err
}
// Fix ownership and mode
err = w.Chown(int(fileUID), int(fileGID))
if err != nil {
return err
}
err = w.Chmod(fileMode)
if err != nil {
return err
}
}
defer logger.WarnOnError(w.Close, "Failed to close file")
// Read the template
tplString, err := os.ReadFile(filepath.Join(d.TemplatesPath(), tpl.Template))
if err != nil {
return fmt.Errorf("Failed to read template file: %w", err)
}
configGet := func(confKey, confDefault *pongo2.Value) *pongo2.Value {
val, ok := d.expandedConfig[confKey.String()]
if !ok {
return confDefault
}
return pongo2.AsValue(strings.TrimRight(val, "\r\n"))
}
err = internalUtil.RenderTemplateFile(w, string(tplString), pongo2.Context{
"trigger": trigger,
"path": tplPath,
"container": containerMeta,
"instance": containerMeta,
"config": d.expandedConfig,
"devices": d.expandedDevices,
"properties": tpl.Properties,
"config_get": configGet,
})
if err != nil {
return fmt.Errorf("Failed to render template: %w", err)
}
return w.Close()
}(tplPath, tpl)
if err != nil {
return err
}
}
return nil
}
// FileSFTPConn returns a connection to the forkfile handler.
func (d *lxc) FileSFTPConn() (net.Conn, error) {
// Lock to avoid concurrent spawning.
spawnUnlock, err := locking.Lock(context.TODO(), fmt.Sprintf("forkfile_%d", d.id))
if err != nil {
return nil, err
}
defer spawnUnlock()
// Create any missing directories in case the instance has never been started before.
err = os.MkdirAll(d.RunPath(), 0o700)
if err != nil {
return nil, err
}
// Trickery to handle paths > 108 chars.
dirFile, err := os.Open(d.RunPath())
if err != nil {
return nil, err
}
defer logger.WarnOnError(dirFile.Close, "Failed to close directory")
forkfileAddr, err := net.ResolveUnixAddr("unix", fmt.Sprintf("/proc/self/fd/%d/forkfile.sock", dirFile.Fd()))
if err != nil {
return nil, err
}
// Attempt to connect on existing socket.
forkfilePath := filepath.Join(d.RunPath(), "forkfile.sock")
forkfileConn, err := net.DialUnix("unix", nil, forkfileAddr)
if err == nil {
// Found an existing server.
return forkfileConn, nil
}
// Check for ongoing operations (that may involve shifting or replacing the root volume) so as to avoid
// allowing SFTP access while the container's filesystem setup is in flux.
// If there is an update operation ongoing and the instance is running then do not wait for the operation
// to complete before continuing as it is possible that a disk device is being removed that requires SFTP
// to clean up the path inside the container. Also it is not possible to be shifting/replacing the root
// volume when the instance is running, so there should be no reason to wait for the operation to finish.
op := operationlock.Get(d.Project().Name, d.Name())
if op.Action() != operationlock.ActionUpdate || !d.IsRunning() {
_ = op.Wait(context.Background())
}
// Setup reverter.
reverter := revert.New()
defer reverter.Fail()
// Create the listener.
_ = os.Remove(forkfilePath)
forkfileListener, err := net.ListenUnix("unix", forkfileAddr)
if err != nil {
return nil, err
}
// Record the socket file identity so cleanup doesn't remove a newer socket.
forkfileInfo, err := os.Stat(forkfilePath)
if err != nil {
_ = forkfileListener.Close()
return nil, err
}
removeForkfileSocket := func() {
info, err := os.Stat(forkfilePath)
if err == nil && os.SameFile(info, forkfileInfo) {
_ = os.Remove(forkfilePath)
}
}
reverter.Add(func() {
_ = forkfileListener.Close()
removeForkfileSocket()
})
// Spawn forkfile in a Go routine.
chReady := make(chan error)
go func() {
// Lock to avoid concurrent running forkfile.
runUnlock, err := locking.Lock(context.TODO(), d.forkfileRunningLockName())
if err != nil {
chReady <- err
return
}
defer runUnlock()
// Mount the filesystem if needed.
if !d.IsRunning() {
// Mount the root filesystem if required.
_, err := d.mount()
if err != nil {
chReady <- err
return
}
defer logger.WarnOnError(d.unmount, "Failed to unmount instance")
}
// Start building the command.
args := []string{
d.state.OS.ExecPath,
"forkfile",
"--",
}
extraFiles := []*os.File{}
// Get the listener file.
forkfileFile, err := forkfileListener.File()
if err != nil {
chReady <- err
return
}
defer logger.WarnOnError(forkfileFile.Close, "Failed to close forkfile listener")
args = append(args, "3")
extraFiles = append(extraFiles, forkfileFile)
// Get the rootfs.
rootfsFile, err := os.Open(d.RootfsPath())
if err != nil {
chReady <- err
return
}
defer logger.WarnOnError(rootfsFile.Close, "Failed to close rootfs")
args = append(args, "4")
extraFiles = append(extraFiles, rootfsFile)
// Get the pidfd if the container is running.
if d.IsRunning() {
pidFd, err := d.InitPidFd()
if err != nil {
chReady <- err
return
}
defer logger.WarnOnError(pidFd.Close, "Failed to close PID fd")
args = append(args, "5")
extraFiles = append(extraFiles, pidFd)
} else {
args = append(args, "-1")
}
// Finalize the args.
args = append(args, fmt.Sprintf("%d", d.InitPID()))
// Prepare sftp server.
forkfile := exec.Cmd{
Path: d.state.OS.ExecPath,
Args: args,
ExtraFiles: extraFiles,
}
var stderr bytes.Buffer
forkfile.Stderr = &stderr
if !d.IsRunning() {
// Get the disk idmap.
idmapset, err := d.DiskIdmap()
if err != nil {
chReady <- err
return
}
if idmapset != nil {
forkfile.SysProcAttr = &syscall.SysProcAttr{
Cloneflags: syscall.CLONE_NEWUSER,
Credential: &syscall.Credential{
Uid: uint32(0),
Gid: uint32(0),
},
UidMappings: idmapset.ToUIDMappings(),
GidMappings: idmapset.ToGIDMappings(),
}
}
}
// Start the server.
err = forkfile.Start()
if err != nil {
chReady <- fmt.Errorf("Failed to run forkfile: %w: %s", err, strings.TrimSpace(stderr.String()))
return
}
// Write PID file.
pidFile := filepath.Join(d.RunPath(), "forkfile.pid")
err = os.WriteFile(pidFile, fmt.Appendf(nil, "%d\n", forkfile.Process.Pid), 0o600)
if err != nil {
chReady <- fmt.Errorf("Failed to write forkfile PID: %w", err)
return
}
// Close the listener and delete the socket immediately after forkfile exits to avoid clients
// thinking a listener is available while other deferred calls are being processed.
defer func() {
_ = forkfileListener.Close()
removeForkfileSocket()
_ = os.Remove(pidFile)
}()
// Indicate the process was spawned without error.
close(chReady)
// Wait for completion.
err = forkfile.Wait()
if err != nil {
d.logger.Error("SFTP server stopped with error", logger.Ctx{"err": err, "stderr": strings.TrimSpace(stderr.String())})
return
}
}()
// Wait for forkfile to have been spawned.
err = <-chReady
if err != nil {
return nil, err
}
// Connect to the new server.
forkfileConn, err = net.DialUnix("unix", nil, forkfileAddr)
if err != nil {
return nil, err
}
// All done.
reverter.Success()
return forkfileConn, nil
}
// FileSFTP returns an SFTP connection to the forkfile handler.
func (d *lxc) FileSFTP() (*sftp.Client, error) {
// Connect to the forkfile daemon.
conn, err := d.FileSFTPConn()
if err != nil {
return nil, err
}
// Get a SFTP client.
client, err := sftp.NewClientPipe(conn, conn)
if err != nil {
_ = conn.Close()
return nil, err
}
go func() {
// Wait for the client to be done before closing the connection.
_ = client.Wait()
_ = conn.Close()
}()
return client, nil
}
// PortForwardConn connects to the given address and TCP port from within the instance's network namespace.
func (d *lxc) PortForwardConn(address string, port int) (net.Conn, error) {
if !d.IsRunning() {
return nil, errors.New("Instance is not running")
}
// Create a socket pair to pass the connection around.
fds, err := unix.Socketpair(unix.AF_UNIX, unix.SOCK_STREAM|unix.SOCK_CLOEXEC, 0)
if err != nil {
return nil, err
}
parentFile := os.NewFile(uintptr(fds[0]), "forknet-parent")
defer func() { _ = parentFile.Close() }()
childFile := os.NewFile(uintptr(fds[1]), "forknet-child")
defer func() { _ = childFile.Close() }()
// Spawn forknet to establish the connection from within the network namespace.
var stderr bytes.Buffer
forknet := exec.Cmd{
Path: d.state.OS.ExecPath,
Args: []string{d.state.OS.ExecPath, "forknet", "connect", "--", fmt.Sprintf("/proc/%d/ns/net", d.InitPID()), address, strconv.Itoa(port)},
ExtraFiles: []*os.File{childFile},
Stderr: &stderr,
}
err = forknet.Run()
if err != nil {
return nil, fmt.Errorf("Failed to run forknet connect: %w: %s", err, strings.TrimSpace(stderr.String()))
}
// Close our copy of the child end so the receive below can't block forever.
_ = childFile.Close()
// Retrieve the connection from forknet.
file, err := netutils.AbstractUnixReceiveFd(int(parentFile.Fd()), netutils.UnixFdsAcceptExact)
if err != nil {
return nil, fmt.Errorf("Failed getting the connection: %w", err)
}
defer func() { _ = file.Close() }()
conn, err := net.FileConn(file)
if err != nil {
return nil, err
}
return conn, nil
}
// stopForkFile attempts to send SIGTERM (if force is true) or SIGINT to forkfile then waits for it to exit.
func (d *lxc) stopForkfile(force bool) {
// Make sure that when the function exits, no forkfile is running by acquiring the lock (which indicates
// that forkfile isn't running and holding the lock) and then releasing it.
defer func() {
unlock, err := locking.Lock(context.TODO(), d.forkfileRunningLockName())
if err != nil {
return
}
unlock()
}()
content, err := os.ReadFile(filepath.Join(d.RunPath(), "forkfile.pid"))
if err != nil {
return
}
pid, err := strconv.ParseInt(strings.TrimSpace(string(content)), 10, 64)
if err != nil {
return
}
d.logger.Debug("Stopping forkfile", logger.Ctx{"pid": pid, "force": force})
if force {
// Forcefully kill the running process.
_ = unix.Kill(int(pid), unix.SIGTERM)
} else {
// Try to send SIGINT to forkfile to indicate it should not accept any new connection.
_ = unix.Kill(int(pid), unix.SIGINT)
}
}
// Console attaches to the instance console.
func (d *lxc) Console(protocol string) (*os.File, chan error, error) {
if protocol != instance.ConsoleTypeConsole {
return nil, nil, fmt.Errorf("Container instances don't support %q output", protocol)
}
chDisconnect := make(chan error, 1)
args := []string{
d.state.OS.ExecPath,
"forkconsole",
project.Instance(d.Project().Name, d.Name()),
d.state.OS.LxcPath,
filepath.Join(d.RunPath(), "lxc.conf"),
"tty=0",
"escape=-1",
}
idmapset, err := d.CurrentIdmap()
if err != nil {
return nil, nil, err
}
var rootUID, rootGID int64
if idmapset != nil {
rootUID, rootGID = idmapset.ShiftIntoNS(0, 0)
}
// Create a PTS pair.
ptx, pty, err := linux.OpenPty(rootUID, rootGID)
if err != nil {
return nil, nil, err
}
// Switch the console file descriptor into raw mode.
_, err = termios.MakeRaw(int(ptx.Fd()))
if err != nil {
return nil, nil, err
}
cmd := exec.Cmd{}
cmd.Path = d.state.OS.ExecPath
cmd.Args = args
cmd.Stdin = pty
cmd.Stdout = pty
cmd.Stderr = pty
err = cmd.Start()
if err != nil {
return nil, nil, err
}
go func() {
err = cmd.Wait()
_ = ptx.Close()
_ = pty.Close()
}()
go func() {
<-chDisconnect
_ = cmd.Process.Kill()
}()
d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceConsole.Event(d, logger.Ctx{"type": instance.ConsoleTypeConsole}))
return ptx, chDisconnect, nil
}
// ConsoleLog returns console log.
func (d *lxc) ConsoleLog(opts liblxc.ConsoleLogOptions) (string, error) {
cc, err := d.initLXC(false)
if err != nil {
return "", err
}
msg, err := cc.ConsoleLog(opts)
if err != nil {
return "", err
}
if opts.ClearLog {
d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceConsoleReset.Event(d, nil))
} else if opts.ReadLog && opts.WriteToLogFile {
d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceConsoleRetrieved.Event(d, nil))
}
return string(msg), nil
}
// Exec executes a command inside the instance.
func (d *lxc) Exec(req api.InstanceExecPost, stdin *os.File, stdout *os.File, stderr *os.File) (instance.Cmd, error) {
// Generate the LXC config if missing.
configPath := filepath.Join(d.RunPath(), "lxc.conf")
if !util.PathExists(configPath) {
cc, err := d.initLXC(true)
if err != nil {
return nil, fmt.Errorf("Load go-lxc struct: %w", err)
}
err = cc.SaveConfigFile(configPath)
if err != nil {
_ = os.Remove(configPath)
return nil, err
}
}
// Prepare the environment
envSlice := []string{}
for k, v := range req.Environment {
envSlice = append(envSlice, fmt.Sprintf("%s=%s", k, v))
}
// Setup logfile
logPath := filepath.Join(d.LogPath(), "forkexec.log")
logFile, err := os.OpenFile(logPath, os.O_WRONLY|os.O_CREATE|os.O_SYNC, 0o644)
if err != nil {
return nil, err
}
defer logger.WarnOnError(logFile.Close, "Failed to close log file")
// Prepare the subcommand
cname := project.Instance(d.Project().Name, d.Name())
args := []string{
d.state.OS.ExecPath,
"forkexec",
cname,
d.state.OS.LxcPath,
filepath.Join(d.RunPath(), "lxc.conf"),
req.Cwd,
fmt.Sprintf("%d", req.User),
fmt.Sprintf("%d", req.Group),
}
args = append(args, "0")
args = append(args, "--")
args = append(args, "env")
args = append(args, envSlice...)
args = append(args, "--")
args = append(args, "cmd")
args = append(args, req.Command...)
cmd := exec.Cmd{}
cmd.Path = d.state.OS.ExecPath
cmd.Args = args
cmd.Stdin = nil
cmd.Stdout = logFile
cmd.Stderr = logFile
// Mitigation for CVE-2019-5736
useRexec := false
if d.expandedConfig["raw.idmap"] != "" {
err := instance.AllowedUnprivilegedOnlyMap(d.expandedConfig["raw.idmap"])
if err != nil {
useRexec = true
}
}
if util.IsTrue(d.expandedConfig["security.privileged"]) {
useRexec = true
}
if useRexec {
cmd.Env = append(os.Environ(), "LXC_MEMFD_REXEC=1")
}
// Setup communication PIPE
rStatus, wStatus, err := os.Pipe()
defer logger.WarnOnError(rStatus.Close, "Failed to close pipe")
if err != nil {
return nil, err
}
cmd.ExtraFiles = []*os.File{stdin, stdout, stderr, wStatus}
err = cmd.Start()
_ = wStatus.Close()
if err != nil {
return nil, err
}
attachedPid := linux.ReadPid(rStatus)
if attachedPid <= 0 {
_ = cmd.Wait()
d.logger.Error("Failed to retrieve PID of executing child process")
return nil, errors.New("Failed to retrieve PID of executing child process")
}
d.logger.Debug("Retrieved PID of executing child process", logger.Ctx{"attachedPid": attachedPid})
d.state.Events.SendLifecycle(d.project.Name, lifecycle.InstanceExec.Event(d, logger.Ctx{"command": req.Command}))
instCmd := &lxcCmd{
cmd: &cmd,
attachedChildPid: int(attachedPid),
}
return instCmd, nil
}
func (d *lxc) cpuStateUsage(cg *cgroup.CGroup) (int64, bool) {
if !cgroup.Supports(cgroup.CPU) {
return -1, false
}
value, err := cg.GetCPUAcctUsage()
if err != nil {
return -1, true
}
return value, true
}
func (d *lxc) cpuState() api.InstanceStateCPU {
cpu := api.InstanceStateCPU{}
cc, err := d.initLXC(false)
if err != nil {
return cpu
}
cg, err := d.cgroup(cc, true)
if err != nil {
return cpu
}
cpuUsage, ok := d.cpuStateUsage(cg)
if ok {
cpu.Usage = cpuUsage
}
cpuCount, err := cg.GetEffectiveCPUs()
if err != nil {
return cpu
}
limitPeriod, limitQuota, err := cg.GetCPUCfsLimit()
if err != nil {
return cpu
}
if limitQuota == -1 {
cpu.AllocatedTime = int64(cpuCount) * 1_000_000_000
} else {
cpu.AllocatedTime = 1_000_000_000 * limitQuota / limitPeriod
}
return cpu
}
func (d *lxc) diskState() map[string]api.InstanceStateDisk {
disk := map[string]api.InstanceStateDisk{}
for _, dev := range d.expandedDevices.Sorted() {
if dev.Config["type"] != "disk" {
continue
}
var usage *storagePools.VolumeUsage
if dev.Config["path"] == "/" {
pool, err := d.getStoragePool()
if err != nil {
d.logger.Error("Error loading storage pool", logger.Ctx{"err": err})
continue
}
usage, err = pool.GetInstanceUsage(d)
if err != nil {
if !errors.Is(err, storageDrivers.ErrNotSupported) {
d.logger.Error("Error getting disk usage", logger.Ctx{"err": err})
}
continue
}
} else if dev.Config["pool"] != "" {
pool, err := storagePools.LoadByName(d.state, dev.Config["pool"])
if err != nil {
d.logger.Error("Error loading storage pool", logger.Ctx{"poolName": dev.Config["pool"], "err": err})
continue
}
volName, _ := internalInstance.SplitVolumeSource(dev.Config["source"])
usage, err = pool.GetCustomVolumeUsage(d.Project().Name, volName)
if err != nil {
if !errors.Is(err, storageDrivers.ErrNotSupported) {
d.logger.Error("Error getting volume usage", logger.Ctx{"volume": dev.Config["source"], "err": err})
}
continue
}
} else {
continue
}
diskState := api.InstanceStateDisk{}
if usage != nil {
diskState.Usage = usage.Used
diskState.Total = usage.Total
}
disk[dev.Name] = diskState
}
return disk
}
func (d *lxc) memoryState() api.InstanceStateMemory {
memory := api.InstanceStateMemory{}
cc, err := d.initLXC(false)
if err != nil {
return memory
}
cg, err := d.cgroup(cc, true)
if err != nil {
return memory
}
if !cgroup.Supports(cgroup.Memory) {
return memory
}
// Memory in bytes
value, err := cg.GetMemoryUsage()
if err == nil {
memory.Usage = value
}
// Memory peak in bytes
value, err = cg.GetMemoryMaxUsage()
if err == nil {
memory.UsagePeak = value
}
// Memory total in bytes
value, err = cg.GetEffectiveMemoryLimit()
if err == nil {
memory.Total = value
}
// Swap in bytes
if memory.Usage > 0 {
value, err := cg.GetMemorySwapUsage()
if err == nil {
memory.SwapUsage = value
}
}
// Swap peak in bytes
if memory.UsagePeak > 0 {
value, err := cg.GetMemorySwapMaxUsage()
if err == nil {
memory.SwapUsagePeak = value
}
}
return memory
}
func (d *lxc) networkState(hostInterfaces []net.Interface) map[string]api.InstanceStateNetwork {
result := map[string]api.InstanceStateNetwork{}
pid := d.InitPID()
if pid < 1 {
return result
}
nw, err := netutils.NetnsGetifaddrs(int32(pid), hostInterfaces)
if err != nil {
d.logger.Error("Failed to retrieve network information via netlink", logger.Ctx{"err": err, "pid": pid})
return result
}
result = nw
// Get host_name from volatile data if not set already.
for name, dev := range result {
if dev.HostName == "" {
dev.HostName = d.localConfig[fmt.Sprintf("volatile.%s.host_name", name)]
result[name] = dev
}
}
return result
}
func (d *lxc) processesState(pid int) (int64, error) {
// Return 0 if not running
if pid == -1 {
return 0, errors.New("PID of LXC instance could not be initialized")
}
cc, err := d.initLXC(false)
if err != nil {
return -1, err
}
cg, err := d.cgroup(cc, true)
if err != nil {
return 0, err
}
if cgroup.Supports(cgroup.Pids) {
value, err := cg.GetProcessesUsage()
if err != nil {
return -1, err
}
return value, nil
}
pids := []int64{int64(pid)}
// Go through the pid list, adding new pids at the end so we go through them all
for i := range pids {
fname := fmt.Sprintf("/proc/%d/task/%d/children", pids[i], pids[i])
fcont, err := os.ReadFile(fname)
if err != nil {
// the process terminated during execution of this loop
continue
}
content := strings.Split(string(fcont), " ")
for j := range content {
pid, err := strconv.ParseInt(content[j], 10, 64)
if err == nil {
pids = append(pids, pid)
}
}
}
return int64(len(pids)), nil
}
// getStorageType returns the storage type of the instance's storage pool.
func (d *lxc) getStorageType() (string, error) {
pool, err := d.getStoragePool()
if err != nil {
return "", err
}
return pool.Driver().Info().Name, nil
}
// mount the instance's rootfs volume if needed.
func (d *lxc) mount() (*storagePools.MountInfo, error) {
pool, err := d.getStoragePool()
if err != nil {
return nil, err
}
if d.IsSnapshot() {
mountInfo, err := pool.MountInstanceSnapshot(d, nil)
if err != nil {
return nil, err
}
return mountInfo, nil
}
mountInfo, err := pool.MountInstance(d, nil)
if err != nil {
return nil, err
}
return mountInfo, nil
}
// unmount the instance's rootfs volume if needed.
func (d *lxc) unmount() error {
pool, err := d.getStoragePool()
if err != nil {
return err
}
if d.IsSnapshot() {
err = pool.UnmountInstanceSnapshot(d, nil)
if err != nil {
return err
}
return nil
}
err = pool.UnmountInstance(d, nil)
if err != nil {
return err
}
return nil
}
// insertMountGo inserts a mount into a container.
// This function is used for the seccomp notifier and so cannot call any
// functions that would cause LXC to talk to the container's monitor. Otherwise
// we'll have a deadlock (with a timeout but still). The InitPID() call here is
// the exception since the seccomp notifier will make sure to always pass a
// valid PID.
func (d *lxc) insertMountGo(source, target, fstype string, flags int, mntnsPID int, idmapType idmap.StorageType) error {
pid := mntnsPID
if pid <= 0 {
// Get the init PID
pid = d.InitPID()
if pid == -1 {
// Container isn't running
return errors.New("Can't insert mount into stopped container")
}
}
// Create the temporary mount target
var tmpMount string
var err error
if internalUtil.IsDir(source) {
tmpMount, err = os.MkdirTemp(d.ShmountsPath(), "incus_mount_")
if err != nil {
return fmt.Errorf("Failed to create shmounts path: %s", err)
}
} else {
f, err := os.CreateTemp(d.ShmountsPath(), "incus_mount_")
if err != nil {
return fmt.Errorf("Failed to create shmounts path: %s", err)
}
tmpMount = f.Name()
_ = f.Close()
}
defer logger.WarnOnError(func() error { return os.Remove(tmpMount) }, "Failed to remove temporary mount")
// Mount the filesystem
err = unix.Mount(source, tmpMount, fstype, uintptr(flags), "")
if err != nil {
return fmt.Errorf("Failed to setup temporary mount: %s", err)
}
defer logger.WarnOnError(func() error { return unix.Unmount(tmpMount, unix.MNT_DETACH) }, "Failed to unmount temporary mount")
// Ensure that only flags modifying mount _properties_ make it through.
// Strip things such as MS_BIND which would cause the creation of a
// shifted mount to be skipped.
// (Fyi, this is just one of the reasons why multiplexers are bad;
// specifically when they do heinous things such as confusing flags
// with commands.)
// This is why multiplexers are bad
shiftfsFlags := (flags & (unix.MS_RDONLY |
unix.MS_NOSUID |
unix.MS_NODEV |
unix.MS_NOEXEC |
unix.MS_DIRSYNC |
unix.MS_NOATIME |
unix.MS_NODIRATIME))
// Move the mount inside the container
mntsrc := filepath.Join("/dev/.incus-mounts", filepath.Base(tmpMount))
pidStr := fmt.Sprintf("%d", pid)
pidFdNr, pidFd, err := seccomp.MakePidFd(pid)
if err != nil {
return err
}
defer logger.WarnOnError(pidFd.Close, "Failed to close PID fd")
if !strings.HasPrefix(target, "/") {
target = "/" + target
}
_, err = subprocess.RunCommandInheritFds(
context.Background(),
[]*os.File{pidFd},
d.state.OS.ExecPath,
"forkmount",
"go-mount",
"--",
pidStr,
fmt.Sprintf("%d", pidFdNr),
mntsrc,
target,
string(idmapType),
fmt.Sprintf("%d", shiftfsFlags),
)
if err != nil {
return err
}
return nil
}
func (d *lxc) insertMountLXC(source, target, fstype string, flags int) error {
cname := project.Instance(d.Project().Name, d.Name())
configPath := filepath.Join(d.RunPath(), "lxc.conf")
if fstype == "" {
fstype = "none"
}
if !strings.HasPrefix(target, "/") {
target = "/" + target
}
_, err := subprocess.RunCommand(
d.state.OS.ExecPath,
"forkmount",
"lxc-mount",
"--",
cname,
d.state.OS.LxcPath,
configPath,
source,
target,
fstype,
fmt.Sprintf("%d", flags),
)
if err != nil {
return err
}
return nil
}
func (d *lxc) moveMount(source, target, fstype string, flags int, idmapType idmap.StorageType) error {
// Get the init PID
pid := d.InitPID()
if pid == -1 {
// Container isn't running
return errors.New("Can't insert mount into stopped container")
}
switch idmapType {
case idmap.StorageTypeIdmapped:
case idmap.StorageTypeNone:
default:
return errors.New("Invalid idmap value specified")
}
pidFdNr, pidFd, err := seccomp.MakePidFd(pid)
if err != nil {
return err
}
defer logger.WarnOnError(pidFd.Close, "Failed to close PID fd")
pidStr := fmt.Sprintf("%d", pid)
if !strings.HasPrefix(target, "/") {
target = "/" + target
}
_, err = subprocess.RunCommandInheritFds(
context.Background(),
[]*os.File{pidFd},
d.state.OS.ExecPath,
"forkmount",
"move-mount",
"--",
pidStr,
fmt.Sprintf("%d", pidFdNr),
fstype,
source,
target,
string(idmapType),
fmt.Sprintf("%d", flags),
)
if err != nil {
return err
}
return nil
}
func (d *lxc) insertMount(source, target, fstype string, flags int, idmapType idmap.StorageType) error {
if idmapType == idmap.StorageTypeIdmapped {
return d.moveMount(source, target, fstype, flags, idmapType)
}
if idmapType == idmap.StorageTypeNone {
return d.insertMountLXC(source, target, fstype, flags)
}
return d.insertMountGo(source, target, fstype, flags, -1, idmapType)
}
func (d *lxc) removeMount(mount string) error {
// Get the init PID
if d.InitPID() == -1 {
// Container isn't running
return errors.New("Can't remove mount from stopped container")
}
configPath := filepath.Join(d.RunPath(), "lxc.conf")
cname := project.Instance(d.Project().Name, d.Name())
if !strings.HasPrefix(mount, "/") {
mount = "/" + mount
}
_, err := subprocess.RunCommand(
d.state.OS.ExecPath,
"forkmount",
"lxc-umount",
"--",
cname,
d.state.OS.LxcPath,
configPath,
mount,
)
if err != nil {
return err
}
return nil
}
// InsertSeccompUnixDevice inserts a seccomp device.
func (d *lxc) InsertSeccompUnixDevice(prefix string, m deviceConfig.Device, pid int) error {
if pid < 0 {
return errors.New("Invalid request PID specified")
}
rootLink := fmt.Sprintf("/proc/%d/root", pid)
rootPath, err := os.Readlink(rootLink)
if err != nil {
return err
}
uid, gid, _, _, err := seccomp.TaskIDs(pid)
if err != nil {
return err
}
idmapset, err := d.CurrentIdmap()
if err != nil {
return err
}
nsuid, nsgid := idmapset.ShiftFromNS(uid, gid)
m["uid"] = fmt.Sprintf("%d", nsuid)
m["gid"] = fmt.Sprintf("%d", nsgid)
if !path.IsAbs(m["path"]) {
cwdLink := fmt.Sprintf("/proc/%d/cwd", pid)
prefixPath, err := os.Readlink(cwdLink)
if err != nil {
return err
}
prefixPath = strings.TrimPrefix(prefixPath, rootPath)
m["path"] = filepath.Join(rootPath, prefixPath, m["path"])
} else {
m["path"] = filepath.Join(rootPath, m["path"])
}
idmapSet, err := d.CurrentIdmap()
if err != nil {
return err
}
dev, err := device.UnixDeviceCreate(d.state, idmapSet, d.DevicesPath(), prefix, m, true)
if err != nil {
return fmt.Errorf("Failed to setup device: %s", err)
}
devPath := dev.HostPath
tgtPath := dev.RelativePath
// Bind-mount it into the container
defer logger.WarnOnError(func() error { return os.Remove(devPath) }, "Failed to remove device path")
return d.insertMountGo(devPath, tgtPath, "none", unix.MS_BIND, pid, idmap.StorageTypeNone)
}
func (d *lxc) removeUnixDevices() error {
// Check that we indeed have devices to remove
if !util.PathExists(d.DevicesPath()) {
return nil
}
// Load the directory listing
dents, err := os.ReadDir(d.DevicesPath())
if err != nil {
return err
}
// Go through all the unix devices
for _, f := range dents {
// Skip non-Unix devices
if !strings.HasPrefix(f.Name(), "forkmknod.unix.") && !strings.HasPrefix(f.Name(), "unix.") && !strings.HasPrefix(f.Name(), "infiniband.unix.") {
continue
}
// Remove the entry
devicePath := filepath.Join(d.DevicesPath(), f.Name())
err := os.Remove(devicePath)
if err != nil {
d.logger.Error("Failed removing unix device", logger.Ctx{"err": err, "path": devicePath})
}
}
return nil
}
// FillNetworkDevice takes a nic or infiniband device type and enriches it with automatically
// generated name and hwaddr properties if these are missing from the device.
func (d *lxc) FillNetworkDevice(name string, m deviceConfig.Device) (deviceConfig.Device, error) {
var err error
newDevice := m.Clone()
// Function to try and guess an available name
nextInterfaceName := func() (string, error) {
devNames := []string{}
// Include all static interface names
for _, dev := range d.expandedDevices.Sorted() {
if dev.Config["name"] != "" && !slices.Contains(devNames, dev.Config["name"]) {
devNames = append(devNames, dev.Config["name"])
}
}
// Include all currently allocated interface names
for k, v := range d.expandedConfig {
if !strings.HasPrefix(k, internalInstance.ConfigVolatilePrefix) {
continue
}
fields := strings.SplitN(k, ".", 3)
if len(fields) != 3 {
continue
}
if fields[2] != "name" || slices.Contains(devNames, v) {
continue
}
devNames = append(devNames, v)
}
// Attempt to include all existing interfaces
cname := project.Instance(d.Project().Name, d.Name())
cc, err := liblxc.NewContainer(cname, d.state.OS.LxcPath)
if err == nil {
defer logger.WarnOnError(cc.Release, "Failed to release container")
interfaces, err := cc.Interfaces()
if err == nil {
for _, name := range interfaces {
if slices.Contains(devNames, name) {
continue
}
devNames = append(devNames, name)
}
}
}
i := 0
name := ""
for {
if m["type"] == "infiniband" {
name = fmt.Sprintf("ib%d", i)
} else {
name = fmt.Sprintf("eth%d", i)
}
// Find a free device name
if !slices.Contains(devNames, name) {
return name, nil
}
i++
}
}
nicType, err := nictype.NICType(d.state, d.Project().Name, m)
if err != nil {
return nil, err
}
isPhysicalWithBridge := device.IsPhysicalNICWithBridge(d.state, d.Project().Name, m)
// Fill in the MAC address.
if (!slices.Contains([]string{"physical", "ipvlan"}, nicType) || isPhysicalWithBridge) && m["hwaddr"] == "" {
configKey := fmt.Sprintf("volatile.%s.hwaddr", name)
volatileHwaddr := d.localConfig[configKey]
if volatileHwaddr == "" {
// Generate a new MAC address.
volatileHwaddr, err = instance.DeviceNextInterfaceHWAddr(d.MACPattern())
if err != nil || volatileHwaddr == "" {
return nil, fmt.Errorf("Failed generating %q: %w", configKey, err)
}
// Update the database and update volatileHwaddr with stored value.
volatileHwaddr, err = d.insertConfigkey(configKey, volatileHwaddr)
if err != nil {
return nil, fmt.Errorf("Failed storing generated config key %q: %w", configKey, err)
}
// Set stored value into current instance config.
d.localConfig[configKey] = volatileHwaddr
d.expandedConfig[configKey] = volatileHwaddr
}
if volatileHwaddr == "" {
return nil, fmt.Errorf("Failed getting %q", configKey)
}
newDevice["hwaddr"] = volatileHwaddr
}
// Fill in the interface name.
if m["name"] == "" {
configKey := fmt.Sprintf("volatile.%s.name", name)
volatileName := d.localConfig[configKey]
if volatileName == "" {
// Generate a new interface name.
volatileName, err = nextInterfaceName()
if err != nil || volatileName == "" {
return nil, fmt.Errorf("Failed generating %q: %w", configKey, err)
}
// Update the database and update volatileName with stored value.
volatileName, err = d.insertConfigkey(configKey, volatileName)
if err != nil {
return nil, fmt.Errorf("Failed storing generated config key %q: %w", configKey, err)
}
// Set stored value into current instance config.
d.localConfig[configKey] = volatileName
d.expandedConfig[configKey] = volatileName
}
if volatileName == "" {
return nil, fmt.Errorf("Failed getting %q", configKey)
}
newDevice["name"] = volatileName
}
return newDevice, nil
}
func (d *lxc) removeDiskDevices() error {
// Check that we indeed have devices to remove
if !util.PathExists(d.DevicesPath()) {
return nil
}
// Load the directory listing
dents, err := os.ReadDir(d.DevicesPath())
if err != nil {
return err
}
// Go through all the unix devices
for _, f := range dents {
// Skip non-disk devices
if !strings.HasPrefix(f.Name(), "disk.") {
continue
}
// Always try to unmount the host side
_ = unix.Unmount(filepath.Join(d.DevicesPath(), f.Name()), unix.MNT_DETACH)
// Remove the entry
diskPath := filepath.Join(d.DevicesPath(), f.Name())
err := os.Remove(diskPath)
if err != nil {
d.logger.Error("Failed to remove disk device path", logger.Ctx{"err": err, "path": diskPath})
}
}
return nil
}
// IsFrozen returns if instance is frozen.
func (d *lxc) IsFrozen() bool {
return d.statusCode() == api.Frozen
}
// IsNesting returns if instance is nested.
func (d *lxc) IsNesting() bool {
return util.IsTrue(d.expandedConfig["security.nesting"])
}
func (d *lxc) isCurrentlyPrivileged() bool {
if !d.IsRunning() {
return d.IsPrivileged()
}
currentIdmap, err := d.CurrentIdmap()
if err != nil {
return d.IsPrivileged()
}
return currentIdmap == nil
}
// IsPrivileged returns if instance is privileged.
func (d *lxc) IsPrivileged() bool {
return util.IsTrue(d.expandedConfig["security.privileged"])
}
// IsRunning returns if instance is running.
func (d *lxc) IsRunning() bool {
return d.isRunningStatusCode(d.statusCode())
}
// CanMigrate returns whether the instance can be migrated.
func (d *lxc) CanMigrate() string {
return d.canMigrate(d)
}
// LockExclusive attempts to get exclusive access to the instance's root volume.
func (d *lxc) LockExclusive() (*operationlock.InstanceOperation, error) {
if d.IsRunning() {
return nil, errors.New("Instance is running")
}
// Prevent concurrent operations the instance.
op, err := operationlock.Create(d.Project().Name, d.Name(), d.op, operationlock.ActionCreate, false, false)
if err != nil {
return nil, err
}
// Stop forkfile as otherwise it will hold the root volume open preventing unmount.
d.stopForkfile(false)
return op, err
}
// InitPID returns PID of init process.
func (d *lxc) InitPID() int {
// Load the go-lxc struct
cc, err := d.initLXC(false)
if err != nil {
return -1
}
return cc.InitPid()
}
// InitPidFd returns pidfd of init process.
func (d *lxc) InitPidFd() (*os.File, error) {
// Load the go-lxc struct
cc, err := d.initLXC(false)
if err != nil {
return nil, err
}
return cc.InitPidFd()
}
// DevptsFd returns dirfd of devpts mount.
func (d *lxc) DevptsFd() (*os.File, error) {
// Load the go-lxc struct
cc, err := d.initLXC(false)
if err != nil {
return nil, err
}
defer d.release()
return cc.DevptsFd()
}
// CurrentIdmap returns current IDMAP.
func (d *lxc) CurrentIdmap() (*idmap.Set, error) {
jsonIdmap, ok := d.LocalConfig()["volatile.idmap.current"]
if !ok {
return d.DiskIdmap()
}
return idmap.NewSetFromJSON(jsonIdmap)
}
// DiskIdmap returns DISK IDMAP.
func (d *lxc) DiskIdmap() (*idmap.Set, error) {
jsonIdmap, ok := d.LocalConfig()["volatile.last_state.idmap"]
if !ok {
return nil, nil
}
return idmap.NewSetFromJSON(jsonIdmap)
}
// NextIdmap returns next IDMAP.
func (d *lxc) NextIdmap() (*idmap.Set, error) {
jsonIdmap, ok := d.LocalConfig()["volatile.idmap.next"]
if !ok {
return d.CurrentIdmap()
}
return idmap.NewSetFromJSON(jsonIdmap)
}
// statusCode returns instance status code.
func (d *lxc) statusCode() api.StatusCode {
// Shortcut to avoid spamming liblxc during ongoing operations.
op := operationlock.Get(d.Project().Name, d.Name())
if op != nil {
if op.Action() == operationlock.ActionStart {
return api.Stopped
}
if op.Action() == operationlock.ActionStop {
if util.IsTrue(d.LocalConfig()["volatile.last_state.ready"]) {
return api.Ready
}
return api.Running
}
}
lxcState, err := d.getLxcState()
if err != nil {
return api.Error
}
statusCode := lxcStatusCode(lxcState)
if statusCode == api.Running && util.IsTrue(d.LocalConfig()["volatile.last_state.ready"]) {
return api.Ready
}
return statusCode
}
// State returns instance state.
func (d *lxc) State() string {
return strings.ToUpper(d.statusCode().String())
}
// LogFilePath log file path.
func (d *lxc) LogFilePath() string {
return filepath.Join(d.LogPath(), "lxc.log")
}
// CGroup returns the cgroup handler for the instance.
func (d *lxc) CGroup() (*cgroup.CGroup, error) {
// Load the go-lxc struct
cc, err := d.initLXC(false)
if err != nil {
return nil, err
}
return d.cgroup(cc, true)
}
func (d *lxc) cgroup(cc *liblxc.Container, running bool) (*cgroup.CGroup, error) {
if cc == nil {
return nil, errors.New("Container not initialized for cgroup")
}
rw := lxcCgroupReadWriter{}
rw.cc = cc
rw.running = running
return cgroup.New(&rw)
}
type lxcCgroupReadWriter struct {
cc *liblxc.Container
running bool
}
// Get reads the value of a cgroup key.
func (rw *lxcCgroupReadWriter) Get(controller string, key string) (string, error) {
if !rw.running {
return strings.Join(rw.cc.ConfigItem(fmt.Sprintf("lxc.cgroup2.%s", key)), "\n"), nil
}
return strings.Join(rw.cc.CgroupItem(key), "\n"), nil
}
// Set writes a value to a cgroup key.
func (rw *lxcCgroupReadWriter) Set(controller string, key string, value string) error {
if !rw.running {
return lxcSetConfigItem(rw.cc, fmt.Sprintf("lxc.cgroup2.%s", key), value)
}
return rw.cc.SetCgroupItem(key, value)
}
// UpdateBackupFile writes the instance's backup.yaml file to storage.
func (d *lxc) UpdateBackupFile() error {
// Prevent concurrent updates to the backup file.
unlock, err := d.updateBackupFileLock(context.Background())
if err != nil {
return err
}
defer unlock()
// Write the current instance state to backup file.
pool, err := d.getStoragePool()
if err != nil {
return err
}
return pool.UpdateInstanceBackupFile(d, true, nil)
}
// Info returns "lxc" and the currently loaded version of LXC.
func (d *lxc) Info() instance.Info {
return instance.Info{
Name: "lxc",
Version: liblxc.Version(),
Type: instancetype.Container,
Error: nil,
}
}
// Metrics returns the metrics set for the instance.
func (d *lxc) Metrics(hostInterfaces []net.Interface) (*metrics.MetricSet, error) {
out := metrics.NewMetricSet(map[string]string{"project": d.project.Name, "name": d.name, "type": instancetype.Container.String()})
if !d.IsRunning() {
return nil, ErrInstanceIsStopped
}
cc, err := d.initLXC(false)
if err != nil {
return nil, err
}
// Load cgroup abstraction
cg, err := d.cgroup(cc, true)
if err != nil {
return nil, err
}
// Get Memory limit.
memoryLimit, err := cg.GetEffectiveMemoryLimit()
if err != nil {
d.logger.Warn("Failed getting effective memory limit", logger.Ctx{"err": err})
}
memoryCached := int64(0)
// Get memory stats.
memStats, err := cg.GetMemoryStats()
if err != nil {
d.logger.Warn("Failed to get memory stats", logger.Ctx{"err": err})
} else {
for k, v := range memStats {
var metricType metrics.MetricType
switch k {
case "active_anon":
metricType = metrics.MemoryActiveAnonBytes
case "active_file":
metricType = metrics.MemoryActiveFileBytes
case "active":
metricType = metrics.MemoryActiveBytes
case "inactive_anon":
metricType = metrics.MemoryInactiveAnonBytes
case "inactive_file":
metricType = metrics.MemoryInactiveFileBytes
case "inactive":
metricType = metrics.MemoryInactiveBytes
case "unevictable":
metricType = metrics.MemoryUnevictableBytes
case "writeback":
metricType = metrics.MemoryWritebackBytes
case "dirty":
metricType = metrics.MemoryDirtyBytes
case "mapped":
metricType = metrics.MemoryMappedBytes
case "rss":
metricType = metrics.MemoryRSSBytes
case "shmem":
metricType = metrics.MemoryShmemBytes
case "cache":
metricType = metrics.MemoryCachedBytes
memoryCached = int64(v)
}
out.AddSamples(metricType, metrics.Sample{Value: float64(v)})
}
}
// Get memory usage.
memoryUsage, err := cg.GetMemoryUsage()
if err != nil {
d.logger.Warn("Failed to get memory usage", logger.Ctx{"err": err})
}
if memoryLimit > 0 {
out.AddSamples(metrics.MemoryMemTotalBytes, metrics.Sample{Value: float64(memoryLimit)})
out.AddSamples(metrics.MemoryMemAvailableBytes, metrics.Sample{Value: float64(memoryLimit - memoryUsage + memoryCached)})
out.AddSamples(metrics.MemoryMemFreeBytes, metrics.Sample{Value: float64(memoryLimit - memoryUsage)})
}
// Get oom kills.
oomKills, err := cg.GetOOMKills()
if err != nil {
d.logger.Warn("Failed to get oom kills", logger.Ctx{"err": err})
}
out.AddSamples(metrics.MemoryOOMKillsTotal, metrics.Sample{Value: float64(oomKills)})
// Handle swap.
if cgroup.Supports(cgroup.MemorySwap) {
swapUsage, err := cg.GetMemorySwapUsage()
if err != nil {
d.logger.Warn("Failed to get swap usage", logger.Ctx{"err": err})
} else {
out.AddSamples(metrics.MemorySwapBytes, metrics.Sample{Value: float64(swapUsage)})
}
}
// Get CPU stats
usage, err := cg.GetCPUAcctUsageAll()
if err != nil {
d.logger.Warn("Failed to get CPU usage", logger.Ctx{"err": err})
} else {
for cpu, stats := range usage {
cpuID := strconv.Itoa(int(cpu))
out.AddSamples(metrics.CPUSecondsTotal, metrics.Sample{Value: float64(stats.System) / 1000000000, Labels: map[string]string{"mode": "system", "cpu": cpuID}})
out.AddSamples(metrics.CPUSecondsTotal, metrics.Sample{Value: float64(stats.User) / 1000000000, Labels: map[string]string{"mode": "user", "cpu": cpuID}})
}
}
// Get CPUs.
CPUs, err := cg.GetEffectiveCPUs()
if err != nil {
d.logger.Warn("Failed to get CPUs", logger.Ctx{"err": err})
} else {
out.AddSamples(metrics.CPUs, metrics.Sample{Value: float64(CPUs)})
}
// Get disk stats
diskStats, err := cg.GetIOStats()
if err != nil {
d.logger.Warn("Failed to get disk stats", logger.Ctx{"err": err})
} else {
for disk, stats := range diskStats {
labels := map[string]string{"device": disk}
out.AddSamples(metrics.DiskReadBytesTotal, metrics.Sample{Value: float64(stats.ReadBytes), Labels: labels})
out.AddSamples(metrics.DiskReadsCompletedTotal, metrics.Sample{Value: float64(stats.ReadsCompleted), Labels: labels})
out.AddSamples(metrics.DiskWrittenBytesTotal, metrics.Sample{Value: float64(stats.WrittenBytes), Labels: labels})
out.AddSamples(metrics.DiskWritesCompletedTotal, metrics.Sample{Value: float64(stats.WritesCompleted), Labels: labels})
}
}
// Get filesystem stats
fsStats, err := d.getFSStats()
if err != nil {
d.logger.Warn("Failed to get fs stats", logger.Ctx{"err": err})
} else {
out.Merge(fsStats)
}
// Get network stats
networkState := d.networkState(hostInterfaces)
for name, netState := range networkState {
labels := map[string]string{"device": name}
out.AddSamples(metrics.NetworkReceiveBytesTotal, metrics.Sample{Value: float64(netState.Counters.BytesReceived), Labels: labels})
out.AddSamples(metrics.NetworkReceivePacketsTotal, metrics.Sample{Value: float64(netState.Counters.PacketsReceived), Labels: labels})
out.AddSamples(metrics.NetworkTransmitBytesTotal, metrics.Sample{Value: float64(netState.Counters.BytesSent), Labels: labels})
out.AddSamples(metrics.NetworkTransmitPacketsTotal, metrics.Sample{Value: float64(netState.Counters.PacketsSent), Labels: labels})
out.AddSamples(metrics.NetworkReceiveErrsTotal, metrics.Sample{Value: float64(netState.Counters.ErrorsReceived), Labels: labels})
out.AddSamples(metrics.NetworkTransmitErrsTotal, metrics.Sample{Value: float64(netState.Counters.ErrorsSent), Labels: labels})
out.AddSamples(metrics.NetworkReceiveDropTotal, metrics.Sample{Value: float64(netState.Counters.PacketsDroppedInbound), Labels: labels})
out.AddSamples(metrics.NetworkTransmitDropTotal, metrics.Sample{Value: float64(netState.Counters.PacketsDroppedOutbound), Labels: labels})
}
// Get number of processes
pids, err := d.processesState(d.InitPID())
if err != nil {
d.logger.Warn("Failed to get total number of processes", logger.Ctx{"err": err})
} else {
out.AddSamples(metrics.ProcsTotal, metrics.Sample{Value: float64(pids)})
}
// Set the timestamps
startedAt, err := d.processStartedAt(d.InitPID())
if err != nil {
d.logger.Warn("Failed to get instance startup time", logger.Ctx{"err": err})
} else {
out.AddSamples(metrics.BootTimeSeconds, metrics.Sample{Value: float64(startedAt.Unix())})
}
out.AddSamples(metrics.TimeSeconds, metrics.Sample{Value: float64(time.Now().Unix())})
return out, nil
}
func (d *lxc) getFSStats() (*metrics.MetricSet, error) {
type mountInfo struct {
Mountpoint string
FSType string
}
out := metrics.NewMetricSet(nil)
mounts, err := os.ReadFile("/proc/mounts")
if err != nil {
return nil, fmt.Errorf("Failed to read /proc/mounts: %w", err)
}
mountMap := make(map[string]mountInfo)
scanner := bufio.NewScanner(bytes.NewReader(mounts))
for scanner.Scan() {
fields := strings.Fields(scanner.Text())
mountMap[fields[0]] = mountInfo{Mountpoint: fields[1], FSType: fields[2]}
}
// Get disk devices
for _, dev := range d.expandedDevices {
if dev["type"] != "disk" || dev["path"] == "" {
continue
}
var statfs *unix.Statfs_t
labels := make(map[string]string)
realDev := ""
if dev["pool"] != "" {
// Expected volume name.
var volName string
var volType storageDrivers.VolumeType
if dev["source"] != "" {
volName = project.StorageVolume(d.project.Name, dev["source"])
volType = storageDrivers.VolumeTypeCustom
} else {
volName = project.Instance(d.project.Name, d.name)
volType = storageDrivers.VolumeTypeContainer
}
// Check that we have a mountpoint.
mountpoint := storageDrivers.GetVolumeMountPath(dev["pool"], volType, volName)
if mountpoint == "" || !util.PathExists(mountpoint) {
continue
}
// Grab the filesystem information.
statfs, err = linux.StatVFS(mountpoint)
if err != nil {
return nil, fmt.Errorf("Failed to stat %s: %w", mountpoint, err)
}
// Grab the pool information to compare.
poolStatfs, err := linux.StatVFS(internalUtil.VarPath("storage-pools", dev["pool"]))
if err != nil {
return nil, fmt.Errorf("Failed to stat %s: %w", mountpoint, err)
}
// Check if we have actual mount-specific information.
if statfs.Type == poolStatfs.Type && statfs.Blocks == poolStatfs.Blocks && statfs.Bfree == poolStatfs.Bfree && statfs.Bavail == poolStatfs.Bavail {
continue
}
// Check if mountPath is in mountMap
isMounted := false
for mountDev, mountInfo := range mountMap {
if mountInfo.Mountpoint != mountpoint {
continue
}
isMounted = true
realDev = mountDev
break
}
if !isMounted {
realDev = dev["source"]
}
} else {
// Skip special disks.
if device.IsSpecialDisk(dev["source"]) {
continue
}
source := dev["source"]
statfs, err = linux.StatVFS(source)
if err != nil {
return nil, fmt.Errorf("Failed to stat %s: %w", dev["source"], err)
}
isMounted := false
// Check if mountPath is in mountMap
for mountDev, mountInfo := range mountMap {
if mountInfo.Mountpoint != source {
continue
}
isMounted = true
stat := unix.Stat_t{}
// Check if dev has a backing file
err = unix.Stat(source, &stat)
if err != nil {
return nil, fmt.Errorf("Failed to stat %s: %w", dev["source"], err)
}
backingFilePath := fmt.Sprintf("/sys/dev/block/%d:%d/loop/backing_file", unix.Major(uint64(stat.Dev)), unix.Minor(uint64(stat.Dev)))
if util.PathExists(backingFilePath) {
// Read backing file
backingFile, err := os.ReadFile(backingFilePath)
if err != nil {
return nil, fmt.Errorf("Failed to read %s: %w", backingFilePath, err)
}
realDev = string(backingFile)
} else {
// Use dev as device
realDev = mountDev
}
break
}
if !isMounted {
realDev = dev["source"]
}
}
// Add labels
labels["device"] = realDev
labels["mountpoint"] = dev["path"]
fsType, err := linux.FSTypeToName(int32(statfs.Type))
if err == nil {
labels["fstype"] = fsType
}
// Add sample
statfsBsize := uint64(statfs.Bsize)
out.AddSamples(metrics.FilesystemSizeBytes, metrics.Sample{Value: float64(statfs.Blocks * statfsBsize), Labels: labels})
out.AddSamples(metrics.FilesystemAvailBytes, metrics.Sample{Value: float64(statfs.Bavail * statfsBsize), Labels: labels})
out.AddSamples(metrics.FilesystemFreeBytes, metrics.Sample{Value: float64(statfs.Bfree * statfsBsize), Labels: labels})
}
return out, nil
}
func (d *lxc) loadRawLXCConfig(cc *liblxc.Container) error {
// Load the LXC raw config.
lxcConfig, ok := d.expandedConfig["raw.lxc"]
if !ok {
return nil
}
// Write to temp config file.
f, err := os.CreateTemp("", "incus_config_")
if err != nil {
return err
}
err = internalIO.WriteAll(f, []byte(lxcConfig))
if err != nil {
return err
}
err = f.Close()
if err != nil {
return err
}
// Load the config.
err = cc.LoadConfigFile(f.Name())
if err != nil {
return fmt.Errorf("Failed to load config file %q: %w", f.Name(), err)
}
_ = os.Remove(f.Name())
return nil
}
// forfileRunningLockName returns the forkfile-running_ID lock name.
func (d *common) forkfileRunningLockName() string {
return fmt.Sprintf("forkfile-running_%d", d.id)
}
// ReloadDevice triggers an empty Update call to the underlying device.
func (d *lxc) ReloadDevice(devName string) error {
dev, err := d.deviceLoad(d, devName, d.expandedDevices[devName], false)
if err != nil {
return err
}
return dev.Update(d.expandedDevices, true)
}
// CanLiveMigrate returns whether the container is live-migratable.
func (d *lxc) CanLiveMigrate() bool {
return util.IsTrue(d.expandedConfig["migration.stateful"])
}
// setupCredentials sets up the systemd credentials directory.
func (d *lxc) setupCredentials(update bool) error {
// Skip updating if the container isn't running.
if update && !d.IsRunning() {
return nil
}
credentialsDir := filepath.Join(d.Path(), "credentials")
credentials := map[string][]byte{}
oldCredentials := map[string]bool{}
var rootUID, rootGID int64
idmapset, err := d.NextIdmap()
if err != nil {
return err
}
if idmapset != nil {
rootUID, rootGID = idmapset.ShiftIntoNS(0, 0)
}
for k, v := range d.expandedConfig {
after, ok := strings.CutPrefix(k, "systemd.credential.")
if ok {
credentials[after] = []byte(v)
continue
}
after, ok = strings.CutPrefix(k, "systemd.credential-binary.")
if ok {
data, err := base64.RawStdEncoding.DecodeString(strings.TrimRight(v, "="))
if err != nil {
return fmt.Errorf("Invalid base64 value for %q: %q", k, v)
}
credentials[after] = data
}
}
// Cleanup the credentials directory.
if update && util.PathExists(credentialsDir) {
credEntries, err := os.ReadDir(credentialsDir)
if err != nil {
return fmt.Errorf("Failed to list credentials directory: %w", err)
}
for _, entry := range credEntries {
oldCredentials[entry.Name()] = true
}
} else {
_ = os.RemoveAll(credentialsDir)
err = internalUtil.MkdirAllOwner(credentialsDir, 0o100, int(rootUID), int(rootGID))
if err != nil {
return fmt.Errorf("Failed to create credentials directory: %w", err)
}
}
credsRoot, err := os.OpenRoot(credentialsDir)
if err != nil {
return fmt.Errorf("Failed to open the credentials directory: %w", err)
}
defer logger.WarnOnError(credsRoot.Close, "Failed to close credentials directory")
for k, v := range credentials {
err := credsRoot.WriteFile(k, v, 0o400)
if err != nil {
return fmt.Errorf("Failed to write credential %q: %w", k, err)
}
err = credsRoot.Chown(k, int(rootUID), int(rootGID))
if err != nil {
return fmt.Errorf("Failed setting permissions for file %q: %w", k, err)
}
delete(oldCredentials, k)
}
for oldCredential := range oldCredentials {
err = os.Remove(filepath.Join(credentialsDir, oldCredential))
if err != nil {
return fmt.Errorf("Failed to remove credential %q: %w", oldCredential, err)
}
}
return nil
}
// GuestOS returns the guest OS. For containers, we can safely assume Linux.
func (d *lxc) GuestOS() osinfo.OSType {
return osinfo.Linux
}
// CreateQcow2Snapshot creates a qcow2 snapshot for a running instance. Not supported by containers.
func (d *lxc) CreateQcow2Snapshot(devPath string, devName string, snapName string, backingFilename string, stateful bool) error {
return instance.ErrNotImplemented
}
// DeleteQcow2Snapshot deletes a qcow2 snapshot for a running instance. Not supported by containers.
func (d *lxc) DeleteQcow2Snapshot(devName string, snapshotIndex int, backingFilename string) error {
return instance.ErrNotImplemented
}
// ExportQcow2Block exports a qcow2 block device. Not supported by containers.
func (d *lxc) ExportQcow2Block(diskName string, diskIndex int) (func(), string, error) {
return nil, "", instance.ErrNotImplemented
}
// ConnectNBD exports a disk over NBD. Not supported by containers.
func (d *lxc) ConnectNBD(diskName string, volSize int64, writable bool) (net.Conn, func(), error) {
return nil, nil, instance.ErrNotImplemented
}
// ConnectNBDAllDisks exports all disks over NBD. Not supported by containers.
func (d *lxc) ConnectNBDAllDisks(reuse bool) (net.Conn, func(), error) {
return nil, nil, instance.ErrNotImplemented
}
// setNICLink sets the link status of the given device.
func (d *lxc) setNICLink(devName string, connected bool, assumeUp bool) error {
// This check is added so that devices that cannot handle link states do not fail to initialize.
if connected && assumeUp {
return nil
}
link, err := ip.LinkByName(d.localConfig["volatile."+devName+".host_name"])
if err != nil {
return fmt.Errorf("Failed to find interface %s: %w", devName, err)
}
if connected {
err = link.SetUp()
if err != nil {
return fmt.Errorf("Failed to bring %s up: %w", devName, err)
}
} else {
err = link.SetDown()
if err != nil {
return fmt.Errorf("Failed to bring %s down: %w", devName, err)
}
}
return nil
}
// CreateBitmap creates a dirty bitmap. Not supported by containers.
func (d *lxc) CreateBitmap(deviceNames []string, data api.StorageVolumeBitmapsPost) error {
return instance.ErrNotImplemented
}
// DeleteBitmap deletes a dirty bitmap. Not supported by containers.
func (d *lxc) DeleteBitmap(deviceName string, bitmapName string) error {
return instance.ErrNotImplemented
}
// GetBitmaps fetches dirty bitmaps. Not supported by containers.
func (d *lxc) GetBitmaps(deviceName string) ([]api.StorageVolumeBitmap, error) {
return nil, instance.ErrNotImplemented
}
Reference in New Issue View Git Blame Copy Permalink