Skip to content

Commit 09459b2

Browse files
ayushr2gvisor-bot
ayushr2
authored and
gvisor-bot
committed
Hide root overlay filestore from container using whiteout.
PiperOrigin-RevId: 507355437
1 parent a53b22a commit 09459b2

File tree

8 files changed

+69
-24
lines changed

8 files changed

+69
-24
lines changed

‎g3doc/user_guide/filesystem.md

+5-5
Original file line numberDiff line numberDiff line change
@@ -48,11 +48,11 @@ layer (tmpfs) be backed by a host file, so all file data is stored on disk.
4848

4949
The newer `--overlay2` flag allows you to achieve these. You can specify
5050
`--overlay2=root:self` in `runtimeArgs`. The overlay backing host file will be
51-
created in the container's root filesystem. Placing the host file in the
52-
container's root filesystem is important because k8s scans the container's root
53-
filesystem from the host to enforce local ephemeral storage limits. You can also
54-
place the overlay host file in another directory using
55-
`--overlay2=root:/path/dir`.
51+
created in the container's root filesystem. This file will be hidden from the
52+
containerized application. Placing the host file in the container's root
53+
filesystem is important because k8s scans the container's root filesystem from
54+
the host to enforce local ephemeral storage limits. You can also place the
55+
overlay host file in another directory using `--overlay2=root:/path/dir`.
5656

5757
## Shared root filesystem
5858

‎pkg/sentry/fsimpl/overlay/filesystem.go

+11-8
Original file line numberDiff line numberDiff line change
@@ -562,16 +562,19 @@ func (fs *filesystem) doCreateAt(ctx context.Context, rp *vfs.ResolvingPath, ct
562562
return nil
563563
}
564564

565+
// CreateWhiteout creates a whiteout at pop. Whiteouts are created with
566+
// character devices with device ID = 0.
567+
//
565568
// Preconditions: pop's parent directory has been copied up.
566-
func (fs *filesystem) createWhiteout(ctx context.Context, vfsObj *vfs.VirtualFilesystem, pop *vfs.PathOperation) error {
567-
return vfsObj.MknodAt(ctx, fs.creds, pop, &vfs.MknodOptions{
569+
func CreateWhiteout(ctx context.Context, vfsObj *vfs.VirtualFilesystem, creds *auth.Credentials, pop *vfs.PathOperation) error {
570+
return vfsObj.MknodAt(ctx, creds, pop, &vfs.MknodOptions{
568571
Mode: linux.S_IFCHR, // permissions == include/linux/fs.h:WHITEOUT_MODE == 0
569572
// DevMajor == DevMinor == 0, from include/linux/fs.h:WHITEOUT_DEV
570573
})
571574
}
572575

573576
func (fs *filesystem) cleanupRecreateWhiteout(ctx context.Context, vfsObj *vfs.VirtualFilesystem, pop *vfs.PathOperation) {
574-
if err := fs.createWhiteout(ctx, vfsObj, pop); err != nil {
577+
if err := CreateWhiteout(ctx, vfsObj, fs.creds, pop); err != nil {
575578
panic(fmt.Sprintf("unrecoverable overlayfs inconsistency: failed to recreate whiteout after failed file creation: %v", err))
576579
}
577580
}
@@ -1237,7 +1240,7 @@ func (fs *filesystem) RenameAt(ctx context.Context, rp *vfs.ResolvingPath, oldPa
12371240
if !whiteoutUpper {
12381241
continue
12391242
}
1240-
if err := fs.createWhiteout(ctx, vfsObj, &vfs.PathOperation{
1243+
if err := CreateWhiteout(ctx, vfsObj, fs.creds, &vfs.PathOperation{
12411244
Root: replaced.upperVD,
12421245
Start: replaced.upperVD,
12431246
Path: fspath.Parse(whiteoutName),
@@ -1320,7 +1323,7 @@ func (fs *filesystem) RenameAt(ctx context.Context, rp *vfs.ResolvingPath, oldPa
13201323
newParent.children[newName] = renamed
13211324
oldParent.dirents = nil
13221325

1323-
if err := fs.createWhiteout(ctx, vfsObj, &oldpop); err != nil {
1326+
if err := CreateWhiteout(ctx, vfsObj, fs.creds, &oldpop); err != nil {
13241327
panic(fmt.Sprintf("unrecoverable overlayfs inconsistency: failed to create whiteout at origin after RenameAt: %v", err))
13251328
}
13261329
if renamed.isDir() {
@@ -1410,7 +1413,7 @@ func (fs *filesystem) RmdirAt(ctx context.Context, rp *vfs.ResolvingPath) error
14101413
if !whiteoutUpper {
14111414
continue
14121415
}
1413-
if err := fs.createWhiteout(ctx, vfsObj, &vfs.PathOperation{
1416+
if err := CreateWhiteout(ctx, vfsObj, fs.creds, &vfs.PathOperation{
14141417
Root: child.upperVD,
14151418
Start: child.upperVD,
14161419
Path: fspath.Parse(whiteoutName),
@@ -1441,7 +1444,7 @@ func (fs *filesystem) RmdirAt(ctx context.Context, rp *vfs.ResolvingPath) error
14411444
return err
14421445
}
14431446
}
1444-
if err := fs.createWhiteout(ctx, vfsObj, &pop); err != nil {
1447+
if err := CreateWhiteout(ctx, vfsObj, fs.creds, &pop); err != nil {
14451448
vfsObj.AbortDeleteDentry(&child.vfsd)
14461449
if child.upperVD.Ok() {
14471450
// Don't attempt to recover from this: the original directory is
@@ -1655,7 +1658,7 @@ func (fs *filesystem) UnlinkAt(ctx context.Context, rp *vfs.ResolvingPath) error
16551658
return err
16561659
}
16571660
}
1658-
if err := fs.createWhiteout(ctx, vfsObj, &pop); err != nil {
1661+
if err := CreateWhiteout(ctx, vfsObj, fs.creds, &pop); err != nil {
16591662
vfsObj.AbortDeleteDentry(&child.vfsd)
16601663
if childLayer == lookupLayerUpper {
16611664
panic(fmt.Sprintf("unrecoverable overlayfs inconsistency: failed to create whiteout after unlinking upper layer file during UnlinkAt: %v", err))

‎runsc/boot/controller.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -438,7 +438,7 @@ func (cm *containerManager) Restore(o *RestoreOpts, _ *struct{}) error {
438438

439439
// Set up the restore environment.
440440
ctx := k.SupervisorContext()
441-
mntr := newContainerMounter(&cm.l.root, cm.l.k, cm.l.mountHints, cm.l.productName)
441+
mntr := newContainerMounter(&cm.l.root, cm.l.k, cm.l.mountHints, cm.l.productName, o.SandboxID)
442442
ctx, err = mntr.configureRestore(ctx)
443443
if err != nil {
444444
return fmt.Errorf("configuring filesystem restore: %v", err)

‎runsc/boot/loader.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -822,7 +822,7 @@ func (l *Loader) createContainerProcess(root bool, cid string, info *containerIn
822822
}
823823
l.startGoferMonitor(cid, int32(info.goferFDs[0].FD()))
824824

825-
mntr := newContainerMounter(info, l.k, l.mountHints, l.productName)
825+
mntr := newContainerMounter(info, l.k, l.mountHints, l.productName, cid)
826826
if root {
827827
if err := mntr.processHints(info.conf, info.procArgs.Credentials); err != nil {
828828
return nil, nil, err

‎runsc/boot/loader_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -417,7 +417,7 @@ func TestCreateMountNamespace(t *testing.T) {
417417
defer l.Destroy()
418418
defer loaderCleanup()
419419

420-
mntr := newContainerMounter(&l.root, l.k, l.mountHints, "")
420+
mntr := newContainerMounter(&l.root, l.k, l.mountHints, "", l.sandboxID)
421421
if err := mntr.processHints(l.root.conf, l.root.procArgs.Credentials); err != nil {
422422
t.Fatalf("failed process hints: %v", err)
423423
}

‎runsc/boot/vfs.go

+30-6
Original file line numberDiff line numberDiff line change
@@ -58,14 +58,22 @@ const (
5858
Nonefs = "none"
5959
)
6060

61+
// SelfOverlayFilestoreDirPrefix is the prefix in the directory name of the
62+
// self overlay filestore directory.
63+
const SelfOverlayFilestoreDirPrefix = ".gvisor.overlay.img."
64+
6165
// SelfOverlayFilestoreDir returns the directory path in which self overlay filestore
6266
// files are stored for a given mount.
6367
func SelfOverlayFilestoreDir(mountSrc, cid string) string {
6468
// We will place filestore files in a gvisor specific hidden directory inside
6569
// the mount being overlayed itself. The same volume can be overlay-ed by
6670
// multiple containers. So make the filestore directory unique to container
6771
// by suffixing the container ID.
68-
return path.Join(mountSrc, ".gvisor.overlay.img."+cid)
72+
return path.Join(mountSrc, selfOverlayFilestoreDirName(cid))
73+
}
74+
75+
func selfOverlayFilestoreDirName(cid string) string {
76+
return SelfOverlayFilestoreDirPrefix + cid
6977
}
7078

7179
// tmpfs has some extra supported options that we must pass through.
@@ -341,9 +349,12 @@ type containerMounter struct {
341349
// productName is the value to show in
342350
// /sys/devices/virtual/dmi/id/product_name.
343351
productName string
352+
353+
// cid is the container ID for the container.
354+
cid string
344355
}
345356

346-
func newContainerMounter(info *containerInfo, k *kernel.Kernel, hints *podMountHints, productName string) *containerMounter {
357+
func newContainerMounter(info *containerInfo, k *kernel.Kernel, hints *podMountHints, productName string, cid string) *containerMounter {
347358
return &containerMounter{
348359
root: info.spec.Root,
349360
mounts: compileMounts(info.spec, info.conf),
@@ -352,6 +363,7 @@ func newContainerMounter(info *containerInfo, k *kernel.Kernel, hints *podMountH
352363
k: k,
353364
hints: hints,
354365
productName: productName,
366+
cid: cid,
355367
}
356368
}
357369

@@ -457,8 +469,7 @@ func (c *containerMounter) createMountNamespace(ctx context.Context, conf *confi
457469
return mns, nil
458470
}
459471

460-
func useOverlayFilestoreFD(conf *config.Config, isDir bool) bool {
461-
overlay2 := conf.GetOverlay2()
472+
func useOverlayFilestoreFD(overlay2 config.Overlay2, isDir bool) bool {
462473
if !overlay2.IsBackedByHostFile() {
463474
return false
464475
}
@@ -473,7 +484,7 @@ func useOverlayFilestoreFD(conf *config.Config, isDir bool) bool {
473484
// layer using tmpfs, and return overlay mount options. "cleanup" must be called
474485
// after the options have been used to mount the overlay, to release refs on
475486
// lower and upper mounts.
476-
func (c *containerMounter) configureOverlay(ctx context.Context, conf *config.Config, creds *auth.Credentials, lowerOpts *vfs.MountOptions, lowerFSName string, useFilestoreFD func(conf *config.Config, isDir bool) bool) (*vfs.MountOptions, func(), error) {
487+
func (c *containerMounter) configureOverlay(ctx context.Context, conf *config.Config, creds *auth.Credentials, lowerOpts *vfs.MountOptions, lowerFSName string, useFilestoreFD func(overlay2 config.Overlay2, isDir bool) bool) (*vfs.MountOptions, func(), error) {
477488
// First copy options from lower layer to upper layer and overlay. Clear
478489
// filesystem specific options.
479490
upperOpts := *lowerOpts
@@ -514,7 +525,8 @@ func (c *containerMounter) configureOverlay(ctx context.Context, conf *config.Co
514525
tmpfsOpts := tmpfs.FilesystemOpts{
515526
RootFileType: uint16(rootType),
516527
}
517-
if useFilestoreFD != nil && useFilestoreFD(conf, rootType == linux.S_IFDIR) {
528+
overlay2 := conf.GetOverlay2()
529+
if useFilestoreFD != nil && useFilestoreFD(overlay2, rootType == linux.S_IFDIR) {
518530
tmpfsOpts.FilestoreFD = c.overlayFilestoreFDs.removeAsFD()
519531
}
520532
upperOpts.GetFilesystemOptions.InternalData = tmpfsOpts
@@ -554,6 +566,18 @@ func (c *containerMounter) configureOverlay(ctx context.Context, conf *config.Co
554566
}
555567
}
556568

569+
// If host filestore is being used and it is backed by self, then we need to
570+
// hide the filestore from the containerized application.
571+
if overlay2.IsBackedBySelf() && useFilestoreFD != nil && useFilestoreFD(overlay2, rootType == linux.S_IFDIR) {
572+
if err := overlay.CreateWhiteout(ctx, c.k.VFS(), creds, &vfs.PathOperation{
573+
Root: upperRootVD,
574+
Start: upperRootVD,
575+
Path: fspath.Parse(selfOverlayFilestoreDirName(c.cid)),
576+
}); err != nil {
577+
return nil, nil, fmt.Errorf("failed to create whiteout to hide self overlay filestore: %w", err)
578+
}
579+
}
580+
557581
// Propagate the lower layer's root's owner, group, and mode to the upper
558582
// layer's root for consistency with VFS1.
559583
err = c.k.VFS().SetStatAt(ctx, creds, &vfs.PathOperation{

‎test/e2e/BUILD

+1
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ go_test(
4242
deps = [
4343
"//pkg/test/dockerutil",
4444
"//pkg/test/testutil",
45+
"//runsc/boot",
4546
"@com_github_docker_docker//api/types/mount:go_default_library",
4647
"@org_golang_x_sys//unix:go_default_library",
4748
],

‎test/e2e/integration_runtime_test.go

+19-2
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ import (
3939
"golang.org/x/sys/unix"
4040
"gvisor.dev/gvisor/pkg/test/dockerutil"
4141
"gvisor.dev/gvisor/pkg/test/testutil"
42+
"gvisor.dev/gvisor/runsc/boot"
4243
)
4344

4445
const (
@@ -186,8 +187,7 @@ func TestOverlayNameTooLong(t *testing.T) {
186187
defer d.CleanUp(ctx)
187188

188189
opts := dockerutil.RunOpts{
189-
Image: "basic/integrationtest",
190-
WorkDir: "/root",
190+
Image: "basic/ubuntu",
191191
}
192192
longName := strings.Repeat("a", unix.NAME_MAX+1)
193193
if got, err := d.Run(ctx, opts, "bash", "-c", fmt.Sprintf("stat %s || true", longName)); err != nil {
@@ -234,3 +234,20 @@ func TestMultipleOverlayMounts(t *testing.T) {
234234
t.Errorf("overlay not applied to both bind mounts, %q file exists", filePath)
235235
}
236236
}
237+
238+
// Tests that the overlay backing host file inside the container's rootfs is
239+
// hidden from the application.
240+
func TestOverlayRootfsWhiteout(t *testing.T) {
241+
ctx := context.Background()
242+
d := dockerutil.MakeContainerWithRuntime(ctx, t, "-overlay")
243+
defer d.CleanUp(ctx)
244+
245+
opts := dockerutil.RunOpts{
246+
Image: "basic/ubuntu",
247+
}
248+
if got, err := d.Run(ctx, opts, "bash", "-c", fmt.Sprintf("ls -al / | grep %q || true", boot.SelfOverlayFilestoreDirPrefix)); err != nil {
249+
t.Fatalf("docker run failed: %s, %v", got, err)
250+
} else if got != "" {
251+
t.Errorf("root directory contains a file/directory whose name contains %q: output = %q", boot.SelfOverlayFilestoreDirPrefix, got)
252+
}
253+
}

0 commit comments

Comments
 (0)