Skip to content
Merged
Show file tree
Hide file tree
Changes from 8 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,9 @@ internal static partial class Fcntl
[LibraryImport(Libraries.SystemNative, EntryPoint = "SystemNative_FcntlSetFD", SetLastError = true)]
internal static partial int SetFD(SafeHandle fd, int flags);

[LibraryImport(Libraries.SystemNative, EntryPoint = "SystemNative_FcntlSetSealWrite", SetLastError = true)]
internal static partial int SetSealWrite(SafeHandle fd);

[LibraryImport(Libraries.SystemNative, EntryPoint = "SystemNative_FcntlGetFD", SetLastError = true)]
internal static partial int GetFD(SafeHandle fd);

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

using System.Runtime.InteropServices;
using System.Threading;
using Microsoft.Win32.SafeHandles;

internal static partial class Interop
{
internal static partial class Sys
{
[LibraryImport(Libraries.SystemNative, EntryPoint = "SystemNative_MemfdCreate", StringMarshalling = StringMarshalling.Utf8, SetLastError = true)]
internal static partial SafeFileHandle MemfdCreate(string name);

[LibraryImport(Libraries.SystemNative, EntryPoint = "SystemNative_MemfdSupported", SetLastError = true)]
private static partial int MemfdSupportedImpl();

private static volatile sbyte s_memfdSupported;

internal static bool IsMemfdSupported
{
get
{
sbyte memfdSupported = s_memfdSupported;
if (memfdSupported == 0)
{
Interlocked.CompareExchange(ref s_memfdSupported, (sbyte)(MemfdSupportedImpl() == 1 ? 1 : -1), 0);
Comment thread
am11 marked this conversation as resolved.
memfdSupported = s_memfdSupported;
}
return memfdSupported > 0;
}
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,8 @@
Link="Common\Interop\Unix\Interop.Libraries.cs" />
<Compile Include="$(CommonPath)Interop\Unix\Interop.Errors.cs"
Link="Common\Interop\Unix\Interop.Errors.cs" />
<Compile Include="$(CommonPath)Interop\Unix\System.Native\Interop.Close.cs"
Link="Common\Interop\Unix\System.Native\Interop.Close.cs" />
<Compile Include="$(CommonPath)Interop\Unix\System.Native\Interop.Fcntl.cs"
Link="Common\Interop\Unix\Interop.Fcntl.cs" />
<Compile Include="$(CommonPath)Interop\Unix\Interop.IOErrors.cs"
Expand Down Expand Up @@ -119,6 +121,8 @@
Link="Common\Interop\Unix\Interop.MAdvise.cs" />
<Compile Include="$(CommonPath)Interop\Unix\System.Native\Interop.ShmOpen.cs"
Link="Common\Interop\Unix\Interop.ShmOpen.cs" />
<Compile Include="$(CommonPath)Interop\Unix\System.Native\Interop.MemfdCreate.cs"
Link="Common\Interop\Unix\Interop.MemfdCreate.cs" />
<Compile Include="$(CommonPath)Interop\Unix\System.Native\Interop.Unlink.cs"
Link="Common\Interop\Unix\Interop.Unlink.cs" />
</ItemGroup>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -162,11 +162,13 @@ private static FileAccess TranslateProtectionsToFileAccess(Interop.Sys.MemoryMap

private static SafeFileHandle CreateSharedBackingObject(Interop.Sys.MemoryMappedProtections protections, long capacity, HandleInheritability inheritability)
{
return CreateSharedBackingObjectUsingMemory(protections, capacity, inheritability)
?? CreateSharedBackingObjectUsingFile(protections, capacity, inheritability);
return Interop.Sys.IsMemfdSupported ?
CreateSharedBackingObjectUsingMemoryMemfdCreate(protections, capacity, inheritability) :
CreateSharedBackingObjectUsingMemoryShmOpen(protections, capacity, inheritability)
?? CreateSharedBackingObjectUsingFile(protections, capacity, inheritability);
}

private static SafeFileHandle? CreateSharedBackingObjectUsingMemory(
private static SafeFileHandle? CreateSharedBackingObjectUsingMemoryShmOpen(
Interop.Sys.MemoryMappedProtections protections, long capacity, HandleInheritability inheritability)
{
// Determine the flags to use when creating the shared memory object
Expand Down Expand Up @@ -244,27 +246,72 @@ private static SafeFileHandle CreateSharedBackingObject(Interop.Sys.MemoryMapped
fd.Dispose();
throw;
}
}

static string GenerateMapName()
private static string GenerateMapName()
{
// macOS shm_open documentation says that the sys-call can fail with ENAMETOOLONG if the name exceeds SHM_NAME_MAX characters.
// The problem is that SHM_NAME_MAX is not defined anywhere and is not consistent amongst macOS versions (arm64 vs x64 for example).
// It was reported in 2008 (https://lists.apple.com/archives/xcode-users/2008/Apr/msg00523.html),
// but considered to be by design (http://web.archive.org/web/20140109200632/http://lists.apple.com/archives/darwin-development/2003/Mar/msg00244.html).
// According to https://github.com/qt/qtbase/blob/1ed449e168af133184633d174fd7339a13d1d595/src/corelib/kernel/qsharedmemory.cpp#L53-L56 the actual value is 30.
// Some other OSS libs use 32 (we did as well, but it was not enough) or 31, but we prefer 30 just to be extra safe.
const int MaxNameLength = 30;
// The POSIX shared memory object name must begin with '/'. After that we just want something short (30) and unique.
const string NamePrefix = "/dotnet_";
return string.Create(MaxNameLength, 0, (span, state) =>
{
// macOS shm_open documentation says that the sys-call can fail with ENAMETOOLONG if the name exceeds SHM_NAME_MAX characters.
// The problem is that SHM_NAME_MAX is not defined anywhere and is not consistent amongst macOS versions (arm64 vs x64 for example).
// It was reported in 2008 (https://lists.apple.com/archives/xcode-users/2008/Apr/msg00523.html),
// but considered to be by design (http://web.archive.org/web/20140109200632/http://lists.apple.com/archives/darwin-development/2003/Mar/msg00244.html).
// According to https://github.com/qt/qtbase/blob/1ed449e168af133184633d174fd7339a13d1d595/src/corelib/kernel/qsharedmemory.cpp#L53-L56 the actual value is 30.
// Some other OSS libs use 32 (we did as well, but it was not enough) or 31, but we prefer 30 just to be extra safe.
const int MaxNameLength = 30;
// The POSIX shared memory object name must begin with '/'. After that we just want something short (30) and unique.
const string NamePrefix = "/dotnet_";
return string.Create(MaxNameLength, 0, (span, state) =>
Span<char> guid = stackalloc char[32];
Guid.NewGuid().TryFormat(guid, out int charsWritten, "N");
Debug.Assert(charsWritten == 32);
NamePrefix.CopyTo(span);
guid.Slice(0, MaxNameLength - NamePrefix.Length).CopyTo(span.Slice(NamePrefix.Length));
Debug.Assert(Encoding.UTF8.GetByteCount(span) <= MaxNameLength); // the standard uses Utf8
});
}

private static SafeFileHandle CreateSharedBackingObjectUsingMemoryMemfdCreate(
Interop.Sys.MemoryMappedProtections protections, long capacity, HandleInheritability inheritability)
{
SafeFileHandle fd = Interop.Sys.MemfdCreate(GenerateMapName());
if (fd.IsInvalid)
{
Interop.ErrorInfo errorInfo = Interop.Sys.GetLastErrorInfo();
fd.Dispose();

throw Interop.GetExceptionForIoErrno(errorInfo);
}

try
{
// Add a write seal for readonly case when readonly protection requested
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Given a request for these permissions, sealing like this makes sense.

But just to make sure I fully understand, would such a request ever be useful? If you can't write to the newly created in-memory-only backing store, what can you do with such a thing?

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah, that's a good point, I was focused on feature parity with shm_open that I didn't looked at the call chain. 😅
I think we can ignore readonly-ness in all three methods called by CreateSharedBackingObject(). If the executable permission is also necessary, we can drop the protection argument altogether + code cleanups?

if ((protections & Interop.Sys.MemoryMappedProtections.PROT_READ) != 0 &&
(protections & Interop.Sys.MemoryMappedProtections.PROT_WRITE) == 0 &&
Interop.Sys.Fcntl.SetSealWrite(fd) == -1)
{
// seal write failed
throw Interop.GetExceptionForIoErrno(Interop.Sys.GetLastErrorInfo());
}

// Give it the right capacity. We do this directly with ftruncate rather
// than via FileStream.SetLength after the FileStream is created because, on some systems,
// lseek fails on shared memory objects, causing the FileStream to think it's unseekable,
// causing it to preemptively throw from SetLength.
Interop.CheckIo(Interop.Sys.FTruncate(fd, capacity));

// SystemNative_MemfdCreate sets CLOEXEC implicitly. If the inheritability requested is Inheritable, remove CLOEXEC.
if (inheritability == HandleInheritability.Inheritable &&
Interop.Sys.Fcntl.SetFD(fd, 0) == -1)
{
Span<char> guid = stackalloc char[32];
Guid.NewGuid().TryFormat(guid, out int charsWritten, "N");
Debug.Assert(charsWritten == 32);
NamePrefix.CopyTo(span);
guid.Slice(0, MaxNameLength - NamePrefix.Length).CopyTo(span.Slice(NamePrefix.Length));
Debug.Assert(Encoding.UTF8.GetByteCount(span) <= MaxNameLength); // the standard uses Utf8
});
throw Interop.GetExceptionForIoErrno(Interop.Sys.GetLastErrorInfo());
}

return fd;
}
catch
{
fd.Dispose();
throw;
}
Comment thread
stephentoub marked this conversation as resolved.
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,6 @@ namespace System
public static partial class Environment
{
public static long WorkingSet =>
(long)(Interop.procfs.TryReadProcessStatusInfo(Interop.procfs.ProcPid.Self, out Interop.procfs.ProcessStatusInfo status) ? status.ResidentSetSize : 0);
(long)(Interop.procfs.TryReadProcessStatusInfo(ProcessId, out Interop.procfs.ProcessStatusInfo status) ? status.ResidentSetSize : 0);
Comment thread
am11 marked this conversation as resolved.
}
}
1 change: 1 addition & 0 deletions src/native/libs/Common/pal_config.h.in
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
#cmakedefine01 HAVE_F_DUPFD
#cmakedefine01 HAVE_F_FULLFSYNC
#cmakedefine01 HAVE_O_CLOEXEC
#cmakedefine01 HAVE_MEMFD_CREATE
#cmakedefine01 HAVE_GETIFADDRS
#cmakedefine01 HAVE_UTSNAME_DOMAINNAME
#cmakedefine01 HAVE_STAT64
Expand Down
3 changes: 3 additions & 0 deletions src/native/libs/System.Native/entrypoints.c
Original file line number Diff line number Diff line change
Expand Up @@ -62,13 +62,16 @@ static const Entry s_sysNative[] =
DllImportEntry(SystemNative_Close)
DllImportEntry(SystemNative_Dup)
DllImportEntry(SystemNative_Unlink)
DllImportEntry(SystemNative_MemfdSupported)
DllImportEntry(SystemNative_MemfdCreate)
DllImportEntry(SystemNative_ShmOpen)
DllImportEntry(SystemNative_ShmUnlink)
DllImportEntry(SystemNative_GetReadDirRBufferSize)
DllImportEntry(SystemNative_ReadDirR)
DllImportEntry(SystemNative_OpenDir)
DllImportEntry(SystemNative_CloseDir)
DllImportEntry(SystemNative_Pipe)
DllImportEntry(SystemNative_FcntlSetSealWrite)
DllImportEntry(SystemNative_FcntlSetFD)
DllImportEntry(SystemNative_FcntlGetFD)
DllImportEntry(SystemNative_FcntlCanGetSetPipeSz)
Expand Down
52 changes: 52 additions & 0 deletions src/native/libs/System.Native/pal_io.c
Original file line number Diff line number Diff line change
Expand Up @@ -369,6 +369,51 @@ int32_t SystemNative_Unlink(const char* path)
return result;
}

int32_t SystemNative_MemfdSupported(void)
{
#if HAVE_MEMFD_CREATE
#ifdef TARGET_LINUX
struct utsname uts;
int32_t major, minor;

// memfd_create is known to only work properly on kernel version > 3.17.
// On earlier versions, it may raise SIGSEGV instead of returning ENOTSUP.
if (uname(&uts) == 0 && sscanf(uts.release, "%d.%d", &major, &minor) == 2 && (major < 3 || (major == 3 && minor < 17)))
{
return 0;
}
#endif

// Note that the name has no affect on file descriptor behavior. From linux manpage:
// Names do not affect the behavior of the file descriptor, and as such multiple files can have the same name without any side effects.
int32_t fd = memfd_create("test", MFD_CLOEXEC | MFD_ALLOW_SEALING);
if (fd < 0) return 0;

close(fd);
return 1;
#else
errno = ENOTSUP;
return 0;
#endif
}

intptr_t SystemNative_MemfdCreate(const char* name)
{
#if HAVE_MEMFD_CREATE
#if defined(SHM_NAME_MAX) // macOS
assert(strlen(name) <= SHM_NAME_MAX);
#elif defined(PATH_MAX) // other Unixes
assert(strlen(name) <= PATH_MAX);
#endif

return memfd_create(name, MFD_CLOEXEC | MFD_ALLOW_SEALING);
Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We can avoid SetSealWrite P/Invoke call if we call fcntl here. I will test.

Current benchmakrs:

Faster base/diff Base Median (ns) Diff Median (ns) Modality
System.IO.MemoryMappedFiles.Tests.Perf_MemoryMappedFile.CreateNew(capacity: 1000 11.10 17470.22 1574.24
System.IO.MemoryMappedFiles.Tests.Perf_MemoryMappedFile.CreateNew(capacity: 1000 10.81 16811.50 1554.71
System.IO.MemoryMappedFiles.Tests.Perf_MemoryMappedFile.CreateNew(capacity: 1000 10.42 16351.91 1568.75
System.IO.MemoryMappedFiles.Tests.Perf_MemoryMappedFile.CreateNew(capacity: 1000 10.28 16327.87 1587.61
System.IO.MemoryMappedFiles.Tests.Perf_MemoryMappedFile.CreateFromFile(capacity: 1.75 50085.65 28573.40
System.IO.MemoryMappedFiles.Tests.Perf_MemoryMappedFile.CreateFromFile(capacity: 1.63 47557.25 29193.17
System.IO.MemoryMappedFiles.Tests.Perf_MemoryMappedFile.CreateFromFile(capacity: 1.62 47405.59 29194.67
System.IO.MemoryMappedFiles.Tests.Perf_MemoryMappedFile.CreateFromFile(capacity: 1.62 47616.49 29359.55

#else
(void)name;
errno = ENOTSUP;
return -1;
#endif
}

intptr_t SystemNative_ShmOpen(const char* name, int32_t flags, int32_t mode)
{
#if defined(SHM_NAME_MAX) // macOS
Expand Down Expand Up @@ -621,6 +666,13 @@ int32_t SystemNative_Pipe(int32_t pipeFds[2], int32_t flags)
return result;
}

int32_t SystemNative_FcntlSetSealWrite(intptr_t fd)
{
int result;
while ((result = fcntl(ToFileDescriptor(fd), F_ADD_SEALS, F_SEAL_WRITE)) < 0 && errno == EINTR);
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

src/native/libs/System.Native/pal_io.c(672,50): error G5EB21622: use of undeclared identifier 'F_ADD_SEALS'

return result;
}

int32_t SystemNative_FcntlSetFD(intptr_t fd, int32_t flags)
{
int result;
Expand Down
21 changes: 21 additions & 0 deletions src/native/libs/System.Native/pal_io.h
Original file line number Diff line number Diff line change
Expand Up @@ -369,6 +369,20 @@ PALEXPORT intptr_t SystemNative_Dup(intptr_t oldfd);
*/
PALEXPORT int32_t SystemNative_Unlink(const char* path);

/**
* Check if the system supports memfd_create(2).
*
* Returns 1 if memfd_create is supported, 0 if not supported, or -1 on failure. Sets errno on failure.
*/
PALEXPORT int32_t SystemNative_MemfdSupported(void);

/**
* Create an anonymous file descriptor. Implemented as shim to memfd_create(2).
*
* Returns file descriptor or -1 on failure. Sets errno on failure.
*/
PALEXPORT intptr_t SystemNative_MemfdCreate(const char* name);

/**
* Open or create a shared memory object. Implemented as shim to shm_open(3).
*
Expand Down Expand Up @@ -418,6 +432,13 @@ PALEXPORT int32_t SystemNative_Pipe(int32_t pipefd[2], // [out] pipefds[0] gets
// for each command. This allows use to have strongly typed arguments and saves
// complexity around converting command codes.

/**
* Sets the write seal on memfd_create file descriptor.
*
* Returns 0 for success; -1 for failure. Sets errno for failure.
*/
PALEXPORT int32_t SystemNative_FcntlSetSealWrite(intptr_t fd);

/**
* Sets the O_CLOEXEC flag on a file descriptor.
*
Expand Down
4 changes: 4 additions & 0 deletions src/native/libs/configure.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,10 @@ check_symbol_exists(
fcntl.h
HAVE_F_FULLFSYNC)

check_function_exists(
memfd_create
HAVE_MEMFD_CREATE)

check_function_exists(
getifaddrs
HAVE_GETIFADDRS)
Expand Down