Skip to content

Add new flags for THMapAllocator #757

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
soumith merged 1 commit into from
Sep 6, 2016
Merged

Add new flags for THMapAllocator #757

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
137 changes: 102 additions & 35 deletions lib/TH/THAllocator.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ struct THMapAllocatorContext_ {
char *filename; /* file name */
int flags;
long size; /* mapped size */
int fd;
};

#define TH_ALLOC_ALIGNMENT 64
Expand All @@ -47,21 +48,35 @@ typedef struct {
int refcount;
} THMapInfo;

char * unknown_filename = "filename not specified";

THMapAllocatorContext *THMapAllocatorContext_new(const char *filename, int flags)
{
THMapAllocatorContext *ctx = THAlloc(sizeof(THMapAllocatorContext));


if (!(flags & TH_ALLOCATOR_MAPPED_SHARED) && !(flags & TH_ALLOCATOR_MAPPED_SHAREDMEM))
flags &= ~TH_ALLOCATOR_MAPPED_NOCREATE;
if ((flags ^ TH_ALLOCATOR_MAPPED_EXCLUSIVE) == 0)
THError("TH_ALLOCATOR_MAPPED_EXCLUSIVE flag requires opening the file "
"in shared mode");

ctx->filename = THAlloc(strlen(filename)+1);
strcpy(ctx->filename, filename);
if (filename) {
ctx->filename = THAlloc(strlen(filename)+1);
strcpy(ctx->filename, filename);
} else {
ctx->filename = unknown_filename;
}
ctx->flags = flags;
ctx->size = 0;
ctx->fd = -1;

return ctx;
}

THMapAllocatorContext *THMapAllocatorContext_newWithFd(const char *filename, int fd, int flags)
{
THMapAllocatorContext *ctx = THMapAllocatorContext_new(filename, flags);
ctx->fd = fd;

return ctx;
}
Expand All @@ -71,14 +86,20 @@ char * THMapAllocatorContext_filename(THMapAllocatorContext *ctx)
return ctx->filename;
}

int THMapAllocatorContext_fd(THMapAllocatorContext *ctx)
{
return ctx->fd;
}

long THMapAllocatorContext_size(THMapAllocatorContext *ctx)
{
return ctx->size;
}

void THMapAllocatorContext_free(THMapAllocatorContext *ctx)
{
THFree(ctx->filename);
if (ctx->filename != unknown_filename)
THFree(ctx->filename);
THFree(ctx);
}

Expand All @@ -98,6 +119,10 @@ static void *_map_alloc(void* ctx_, long size)
THError("exclusive file mapping is not supported on Windows");
if (ctx->flags & TH_ALLOCATOR_MAPPED_NOCREATE)
THError("file mapping without creation is not supported on Windows");
if (ctx->flags & TH_ALLOCATOR_MAPPED_KEEPFD)
THError("TH_ALLOCATOR_MAPPED_KEEPFD not supported on Windows");
if (ctx->flags & TH_ALLOCATOR_MAPPED_FROMFD)
THError("TH_ALLOCATOR_MAPPED_FROMFD not supported on Windows");

/* open file */
/* FILE_FLAG_RANDOM_ACCESS ? */
Expand Down Expand Up @@ -193,9 +218,9 @@ static void *_map_alloc(void* ctx_, long size)
/* open file */
int fd;
int flags;
long fdsz;
struct stat file_stat;

if (ctx->flags)
if (ctx->flags & (TH_ALLOCATOR_MAPPED_SHARED | TH_ALLOCATOR_MAPPED_SHAREDMEM))
flags = O_RDWR | O_CREAT;
else
flags = O_RDONLY;
Expand All @@ -205,35 +230,40 @@ static void *_map_alloc(void* ctx_, long size)
if (ctx->flags & TH_ALLOCATOR_MAPPED_NOCREATE)
flags &= ~O_CREAT;

if(ctx->flags & TH_ALLOCATOR_MAPPED_SHARED)
{
if((fd = open(ctx->filename, flags, (mode_t)0600)) == -1)
THError("unable to open file <%s> in read-write mode", ctx->filename);
}
else if (ctx->flags & TH_ALLOCATOR_MAPPED_SHAREDMEM)
{
if (!(ctx->flags & TH_ALLOCATOR_MAPPED_FROMFD)) {
if(ctx->flags & TH_ALLOCATOR_MAPPED_SHARED)
{
if((fd = open(ctx->filename, flags, (mode_t)0600)) == -1)
THError("unable to open file <%s> in read-write mode", ctx->filename);
}
else if (ctx->flags & TH_ALLOCATOR_MAPPED_SHAREDMEM)
{
#ifdef HAVE_SHM_OPEN
if((fd = shm_open(ctx->filename, flags, (mode_t)0600)) == -1)
THError("unable to open shared memory object <%s> in read-write mode", ctx->filename);
if((fd = shm_open(ctx->filename, flags, (mode_t)0600)) == -1)
THError("unable to open shared memory object <%s> in read-write mode", ctx->filename);
#else
THError("unable to open file <%s> in sharedmem mode, shm_open unavailable on this platform");
THError("unable to open file <%s> in sharedmem mode, shm_open unavailable on this platform");
#endif
}
else
{
if((fd = open(ctx->filename, O_RDONLY)) == -1)
THError("unable to open file <%s> in read-only mode", ctx->filename);
}
else
{
if((fd = open(ctx->filename, O_RDONLY)) == -1)
THError("unable to open file <%s> in read-only mode", ctx->filename);
}
} else {
fd = ctx->fd;
}

if((fdsz = lseek(fd, 0, SEEK_END)) == -1)
if(fstat(fd, &file_stat) == -1)
{
close(fd);
THError("unable to seek at end of file <%s>", ctx->filename);
if (!(ctx->flags & TH_ALLOCATOR_MAPPED_FROMFD))
close(fd);
THError("unable to stat the file <%s>", ctx->filename);
}

if(size > 0)
{
if(size > fdsz)
if(size > file_stat.st_size)
{
if(ctx->flags)
{
Expand All @@ -243,7 +273,7 @@ static void *_map_alloc(void* ctx_, long size)
if(ftruncate(fd, size) == -1)
THError("unable to resize shared memory file <%s> to the right size", ctx->filename);
}
if((fdsz = lseek(fd, size-1, SEEK_SET)) == -1)
if(fstat(fd, &file_stat) == -1 || file_stat.st_size < size)
{
close(fd);
THError("unable to stretch file <%s> to the right size", ctx->filename);
Expand All @@ -262,18 +292,40 @@ static void *_map_alloc(void* ctx_, long size)
}
}
else
size = fdsz;
size = file_stat.st_size;

ctx->size = size; /* if we are here, it must be the right size */

/* map it */
if(ctx->flags)
if (ctx->flags & (TH_ALLOCATOR_MAPPED_SHARED | TH_ALLOCATOR_MAPPED_SHAREDMEM))
data = mmap(NULL, ctx->size, PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0);
else
data = mmap(NULL, ctx->size, PROT_READ|PROT_WRITE, MAP_PRIVATE, fd, 0);

if(close(fd) == -1)
THError("Error closing file <%s>", ctx->filename);
if (ctx->flags & TH_ALLOCATOR_MAPPED_KEEPFD) {
ctx->fd = fd;
} else {
if(close(fd) == -1)
THError("Error closing file <%s>", ctx->filename);
ctx->fd = -1;
}

if (ctx->flags & TH_ALLOCATOR_MAPPED_UNLINK) {
if (ctx->flags & TH_ALLOCATOR_MAPPED_SHAREDMEM)
{
#ifdef HAVE_SHM_UNLINK
if (shm_unlink(ctx->filename) == -1)
THError("could not unlink the shared memory file %s", ctx->filename);
#else
THError("could not unlink the shared memory file %s, shm_unlink not available on platform", ctx->filename);
#endif
}
else
{
if (unlink(ctx->filename) == -1)
THError("could not unlink file %s", ctx->filename);
}
}

if(data == MAP_FAILED)
{
Expand Down Expand Up @@ -302,16 +354,25 @@ static void THMapAllocator_free(void* ctx_, void* data) {
if(!UnmapViewOfFile((LPINT)data))
THError("could not unmap the shared memory file");
#else /* _WIN32 */
if (ctx->flags & TH_ALLOCATOR_MAPPED_KEEPFD) {
if (close(ctx->fd) == -1)
THError("could not close file descriptor %d", ctx->fd);
}

if (munmap(data, ctx->size))
THError("could not unmap the shared memory file");
if (ctx->flags & TH_ALLOCATOR_MAPPED_SHAREDMEM)

if (!(ctx->flags & (TH_ALLOCATOR_MAPPED_FROMFD | TH_ALLOCATOR_MAPPED_UNLINK)))
{
if (ctx->flags & TH_ALLOCATOR_MAPPED_SHAREDMEM)
{
#ifdef HAVE_SHM_UNLINK
if (shm_unlink(ctx->filename) == -1)
THError("could not unlink the shared memory file %s", ctx->filename);
if (shm_unlink(ctx->filename) == -1)
THError("could not unlink the shared memory file %s", ctx->filename);
#else
THError("could not unlink the shared memory file %s, shm_unlink not available on platform", ctx->filename);
THError("could not unlink the shared memory file %s, shm_unlink not available on platform", ctx->filename);
#endif
}
}
#endif /* _WIN32 */

Expand Down Expand Up @@ -350,8 +411,14 @@ static void THMapAllocator_free(void* ctx, void* data) {
static void * THRefcountedMapAllocator_alloc(void *_ctx, long size) {
THMapAllocatorContext *ctx = _ctx;

if (ctx->flags & TH_ALLOCATOR_MAPPED_FROMFD)
THError("THRefcountedMapAllocator doesn't support TH_ALLOCATOR_MAPPED_FROMFD flag");
if (ctx->flags & TH_ALLOCATOR_MAPPED_KEEPFD)
THError("THRefcountedMapAllocator doesn't support TH_ALLOCATOR_MAPPED_KEEPFD flag");
if (ctx->flags & TH_ALLOCATOR_MAPPED_UNLINK)
THError("THRefcountedMapAllocator doesn't support TH_ALLOCATOR_MAPPED_UNLINK flag");
if (!(ctx->flags & TH_ALLOCATOR_MAPPED_SHAREDMEM))
THError("THRefcountedMapAllcator requires SHAREDMEM flag");
THError("THRefcountedMapAllocator requires TH_ALLOCATOR_MAPPED_SHAREDMEM flag");

size = size + TH_ALLOC_ALIGNMENT;
void *ptr = _map_alloc(ctx, size);
Expand Down
6 changes: 6 additions & 0 deletions lib/TH/THAllocator.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,9 @@
#define TH_ALLOCATOR_MAPPED_SHAREDMEM 2
#define TH_ALLOCATOR_MAPPED_EXCLUSIVE 4
#define TH_ALLOCATOR_MAPPED_NOCREATE 8
#define TH_ALLOCATOR_MAPPED_KEEPFD 16
#define TH_ALLOCATOR_MAPPED_FROMFD 32
#define TH_ALLOCATOR_MAPPED_UNLINK 64

/* Custom allocator
*/
Expand All @@ -25,7 +28,10 @@ extern THAllocator THDefaultAllocator;
*/
typedef struct THMapAllocatorContext_ THMapAllocatorContext;
TH_API THMapAllocatorContext *THMapAllocatorContext_new(const char *filename, int flags);
TH_API THMapAllocatorContext *THMapAllocatorContext_newWithFd(const char *filename,
int fd, int flags);
TH_API char * THMapAllocatorContext_filename(THMapAllocatorContext *ctx);
TH_API int THMapAllocatorContext_fd(THMapAllocatorContext *ctx);
TH_API long THMapAllocatorContext_size(THMapAllocatorContext *ctx);
TH_API void THMapAllocatorContext_free(THMapAllocatorContext *ctx);

Expand Down