Commit 6dfcce34 authored by Libor Peltan's avatar Libor Peltan Committed by Daniel Salzman

zscanner: support for loading zone from non-normal file (e.g. pipe)

...by reading it whole into dynamically growing memory buffer
parent 93dc3f4a
......@@ -37,10 +37,6 @@ static const err_table_t err_msgs[] = {
"file open error" ),
ERR_ITEM( ZS_FILE_INVALID,
"invalid file" ),
ERR_ITEM( ZS_FILE_PATH,
"file path error" ),
ERR_ITEM( ZS_FILE_MMAP,
"file memory mapping error" ),
ERR_ITEM( ZS_DOS_NEWLINE,
"unsupported CRLF newline, remove CR bytes" ),
ERR_ITEM( ZS_UNCOVERED_STATE,
......
......@@ -30,8 +30,6 @@ enum err_codes {
ZS_ENOMEM,
ZS_FILE_OPEN,
ZS_FILE_INVALID,
ZS_FILE_PATH,
ZS_FILE_MMAP,
ZS_DOS_NEWLINE,
ZS_UNCOVERED_STATE,
ZS_UNCLOSED_MULTILINE,
......
......@@ -163,8 +163,12 @@ static void input_deinit(
if (s->file.descriptor != -1) {
// Unmap the file content.
if (s->input.start != NULL) {
munmap((void *)s->input.start,
s->input.end - s->input.start);
if (s->input.mmaped) {
munmap((void *)s->input.start,
s->input.end - s->input.start);
} else {
free((void *)s->input.start);
}
}
// Close the opened file.
......@@ -224,6 +228,32 @@ static int set_input_string(
return 0;
}
static char *read_file_to_buf(
int fd,
size_t *bufsize)
{
size_t bufs = 0, newbufs = 8192;
char *buf = malloc(bufs + newbufs);
int ret = 0;
while (buf != NULL && (ret = read(fd, buf + bufs, newbufs)) == newbufs) {
bufs += newbufs;
newbufs = bufs;
char *newbuf = realloc(buf, bufs + newbufs);
if (newbuf == NULL) {
free(buf);
}
buf = newbuf;
}
if (ret < 0) {
free(buf);
return NULL;
}
*bufsize = bufs + ret;
return buf;
}
__attribute__((visibility("default")))
int zs_set_input_string(
zs_scanner_t *s,
......@@ -257,36 +287,37 @@ int zs_set_input_file(
return -1;
}
char *start = NULL;
size_t size = 0;
// Check for regular file input.
struct stat file_stat;
if (fstat(s->file.descriptor, &file_stat) == -1 ||
!S_ISREG(file_stat.st_mode)) {
ERR(ZS_FILE_INVALID);
input_deinit(s, false);
return -1;
}
!S_ISREG(file_stat.st_mode) ||
(start = mmap(0, file_stat.st_size, PROT_READ, MAP_SHARED,
s->file.descriptor, 0)) == MAP_FAILED) {
// Check for empty file (cannot mmap).
if (file_stat.st_size > 0) {
// Map the file to the memory.
char *start = mmap(0, file_stat.st_size, PROT_READ, MAP_SHARED,
s->file.descriptor, 0);
if (start == MAP_FAILED) {
ERR(ZS_FILE_MMAP);
// Workaround if cannot mmap, read to memory.
start = read_file_to_buf(s->file.descriptor, &size);
if (start == NULL) {
ERR(ZS_FILE_INVALID);
input_deinit(s, false);
return -1;
}
} else if (file_stat.st_size > 0) { // Skip an empty file.
size = file_stat.st_size;
s->input.mmaped = true;
// Try to set the mapped memory advise to sequential.
(void)madvise(start, file_stat.st_size, MADV_SEQUENTIAL);
// Set the scanner input limits.
s->input.start = start;
s->input.current = start;
s->input.end = start + file_stat.st_size;
(void)madvise(start, size, MADV_SEQUENTIAL);
}
// Get absolute path of the zone file.
// Set the scanner input limits.
s->input.start = start;
s->input.current = start;
s->input.end = start + size;
// Get absolute path of the zone file if possible.
char *full_name = realpath(file_name, NULL);
if (full_name != NULL) {
free(s->path);
......@@ -297,10 +328,6 @@ int zs_set_input_file(
input_deinit(s, false);
return -1;
}
} else {
ERR(ZS_FILE_PATH);
input_deinit(s, false);
return -1;
}
s->file.name = strdup(file_name);
......@@ -5526,8 +5526,12 @@ static void input_deinit(
if (s->file.descriptor != -1) {
// Unmap the file content.
if (s->input.start != NULL) {
munmap((void *)s->input.start,
s->input.end - s->input.start);
if (s->input.mmaped) {
munmap((void *)s->input.start,
s->input.end - s->input.start);
} else {
free((void *)s->input.start);
}
}
// Close the opened file.
......@@ -5587,6 +5591,32 @@ static int set_input_string(
return 0;
}
static char *read_file_to_buf(
int fd,
size_t *bufsize)
{
size_t bufs = 0, newbufs = 8192;
char *buf = malloc(bufs + newbufs);
int ret = 0;
while (buf != NULL && (ret = read(fd, buf + bufs, newbufs)) == newbufs) {
bufs += newbufs;
newbufs = bufs;
char *newbuf = realloc(buf, bufs + newbufs);
if (newbuf == NULL) {
free(buf);
}
buf = newbuf;
}
if (ret < 0) {
free(buf);
return NULL;
}
*bufsize = bufs + ret;
return buf;
}
__attribute__((visibility("default")))
int zs_set_input_string(
zs_scanner_t *s,
......@@ -5620,36 +5650,37 @@ int zs_set_input_file(
return -1;
}
char *start = NULL;
size_t size = 0;
// Check for regular file input.
struct stat file_stat;
if (fstat(s->file.descriptor, &file_stat) == -1 ||
!S_ISREG(file_stat.st_mode)) {
ERR(ZS_FILE_INVALID);
input_deinit(s, false);
return -1;
}
// Check for empty file (cannot mmap).
if (file_stat.st_size > 0) {
// Map the file to the memory.
char *start = mmap(0, file_stat.st_size, PROT_READ, MAP_SHARED,
s->file.descriptor, 0);
if (start == MAP_FAILED) {
ERR(ZS_FILE_MMAP);
!S_ISREG(file_stat.st_mode) ||
(start = mmap(0, file_stat.st_size, PROT_READ, MAP_SHARED,
s->file.descriptor, 0)) == MAP_FAILED) {
// Workaround if cannot mmap, read to memory.
start = read_file_to_buf(s->file.descriptor, &size);
if (start == NULL) {
ERR(ZS_FILE_INVALID);
input_deinit(s, false);
return -1;
}
} else if (file_stat.st_size > 0) { // Skip an empty file.
size = file_stat.st_size;
s->input.mmaped = true;
// Try to set the mapped memory advise to sequential.
(void)madvise(start, file_stat.st_size, MADV_SEQUENTIAL);
// Set the scanner input limits.
s->input.start = start;
s->input.current = start;
s->input.end = start + file_stat.st_size;
(void)madvise(start, size, MADV_SEQUENTIAL);
}
// Get absolute path of the zone file.
// Set the scanner input limits.
s->input.start = start;
s->input.current = start;
s->input.end = start + size;
// Get absolute path of the zone file if possible.
char *full_name = realpath(file_name, NULL);
if (full_name != NULL) {
free(s->path);
......@@ -5660,10 +5691,6 @@ int zs_set_input_file(
input_deinit(s, false);
return -1;
}
} else {
ERR(ZS_FILE_PATH);
input_deinit(s, false);
return -1;
}
s->file.name = strdup(file_name);
......
......@@ -193,6 +193,8 @@ struct zs_scanner {
const char *end;
/*! Indication for the final block parsing. */
bool eof;
/*! Indication of being mmap()-ed (malloc()-ed otherwise). */
bool mmaped;
} input;
/*! File input parameters. */
......
......@@ -164,8 +164,12 @@ static void input_deinit(
if (s->file.descriptor != -1) {
// Unmap the file content.
if (s->input.start != NULL) {
munmap((void *)s->input.start,
s->input.end - s->input.start);
if (s->input.mmaped) {
munmap((void *)s->input.start,
s->input.end - s->input.start);
} else {
free((void *)s->input.start);
}
}
// Close the opened file.
......@@ -225,6 +229,32 @@ static int set_input_string(
return 0;
}
static char *read_file_to_buf(
int fd,
size_t *bufsize)
{
size_t bufs = 0, newbufs = 8192;
char *buf = malloc(bufs + newbufs);
int ret = 0;
while (buf != NULL && (ret = read(fd, buf + bufs, newbufs)) == newbufs) {
bufs += newbufs;
newbufs = bufs;
char *newbuf = realloc(buf, bufs + newbufs);
if (newbuf == NULL) {
free(buf);
}
buf = newbuf;
}
if (ret < 0) {
free(buf);
return NULL;
}
*bufsize = bufs + ret;
return buf;
}
__attribute__((visibility("default")))
int zs_set_input_string(
zs_scanner_t *s,
......@@ -258,36 +288,37 @@ int zs_set_input_file(
return -1;
}
char *start = NULL;
size_t size = 0;
// Check for regular file input.
struct stat file_stat;
if (fstat(s->file.descriptor, &file_stat) == -1 ||
!S_ISREG(file_stat.st_mode)) {
ERR(ZS_FILE_INVALID);
input_deinit(s, false);
return -1;
}
// Check for empty file (cannot mmap).
if (file_stat.st_size > 0) {
// Map the file to the memory.
char *start = mmap(0, file_stat.st_size, PROT_READ, MAP_SHARED,
s->file.descriptor, 0);
if (start == MAP_FAILED) {
ERR(ZS_FILE_MMAP);
!S_ISREG(file_stat.st_mode) ||
(start = mmap(0, file_stat.st_size, PROT_READ, MAP_SHARED,
s->file.descriptor, 0)) == MAP_FAILED) {
// Workaround if cannot mmap, read to memory.
start = read_file_to_buf(s->file.descriptor, &size);
if (start == NULL) {
ERR(ZS_FILE_INVALID);
input_deinit(s, false);
return -1;
}
} else if (file_stat.st_size > 0) { // Skip an empty file.
size = file_stat.st_size;
s->input.mmaped = true;
// Try to set the mapped memory advise to sequential.
(void)madvise(start, file_stat.st_size, MADV_SEQUENTIAL);
// Set the scanner input limits.
s->input.start = start;
s->input.current = start;
s->input.end = start + file_stat.st_size;
(void)madvise(start, size, MADV_SEQUENTIAL);
}
// Get absolute path of the zone file.
// Set the scanner input limits.
s->input.start = start;
s->input.current = start;
s->input.end = start + size;
// Get absolute path of the zone file if possible.
char *full_name = realpath(file_name, NULL);
if (full_name != NULL) {
free(s->path);
......@@ -298,10 +329,6 @@ int zs_set_input_file(
input_deinit(s, false);
return -1;
}
} else {
ERR(ZS_FILE_PATH);
input_deinit(s, false);
return -1;
}
s->file.name = strdup(file_name);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment