Use strings to identify section types in the on-disk repo format.

Previously, a given razor file type had a fixed number of sections in a
fixed order, identified by an integer type. Now, sections are identified
by a named string (stored in a string pool after the section lists).

This will allow for razor files to contain arbitrary sections.

For bonus points, also drop the 4k section alignment and change the
magic byte string to "RZDB".

3 files changed, 116 insertions, 140 deletions

diff --git a/TODO b/TODO
index 46cedcb..93c1f75 100644
--- a/TODO
+++ b/TODO
@@ -118,14 +118,7 @@ Towards replacing rpm + yum (0.1):
 
 Package set file format items:
 
-- drop the 4k section alignment
-
-- just use strings for header identifiers, make the string pool
-  section have a fixed string (maybe make "strings" always the first
-  string so its index is 0), or maybe just require that it's the first
-  section in the file.
-
-- nail down byte-order of rzdb file.
+- nail down byte-order of repo file.
 
 - version the sections in the file, put the element size in the header
   so we can add stuff to elements in a backwards compatible way.
diff --git a/librazor/razor-internal.h b/librazor/razor-internal.h
index c47e4f1..b82a468 100644
--- a/librazor/razor-internal.h
+++ b/librazor/razor-internal.h
@@ -82,7 +82,7 @@ uint32_t hashtable_tokenize(struct hashtable *table, const char *string);
 
 
 struct razor_set_section {
-	uint32_t type;
+	uint32_t name;
 	uint32_t offset;
 	uint32_t size;
 };
@@ -90,25 +90,23 @@ struct razor_set_section {
 struct razor_set_header {
 	uint32_t magic;
 	uint32_t version;
-	struct razor_set_section sections[0];
+	uint32_t num_sections;
 };
 
-#define RAZOR_MAGIC 		0x7a7a7a7a
-#define RAZOR_DETAILS_MAGIC 	0x7a7a7a7b
-#define RAZOR_FILES_MAGIC 	0x7a7a7a7c
-#define RAZOR_VERSION 1
+#define RAZOR_MAGIC 	0x525a4442
+#define RAZOR_VERSION	1
 
-#define RAZOR_STRING_POOL		0
-#define RAZOR_PACKAGES			1
-#define RAZOR_PROPERTIES		2
-#define RAZOR_PACKAGE_POOL		3
-#define RAZOR_PROPERTY_POOL		4
+#define RAZOR_STRING_POOL		"string_pool"
+#define RAZOR_PACKAGES			"packages"
+#define RAZOR_PROPERTIES		"properties"
+#define RAZOR_PACKAGE_POOL		"package_pool"
+#define RAZOR_PROPERTY_POOL		"property_pool"
 
-#define RAZOR_DETAILS_STRING_POOL	0
+#define RAZOR_DETAILS_STRING_POOL	"details_string_pool"
 
-#define RAZOR_FILES			0
-#define RAZOR_FILE_POOL			1
-#define RAZOR_FILE_STRING_POOL		2
+#define RAZOR_FILES			"files"
+#define RAZOR_FILE_POOL			"file_pool"
+#define RAZOR_FILE_STRING_POOL		"file_string_pool"
 
 struct razor_package {
 	uint name  : 24;
@@ -150,9 +148,15 @@ struct razor_set {
  	struct array file_pool;
 	struct array file_string_pool;
 	struct array details_string_pool;
+
 	struct razor_set_header *header;
+	size_t header_size;
+
 	struct razor_set_header *details_header;
+	size_t details_header_size;
+
 	struct razor_set_header *files_header;
+	size_t files_header_size;
 };
 
 struct import_entry {
diff --git a/librazor/razor.c b/librazor/razor.c
index 4276381..6ed90c7 100644
--- a/librazor/razor.c
+++ b/librazor/razor.c
@@ -49,7 +49,12 @@ zalloc(size_t size)
 	return p;
 }
 
-struct razor_set_section razor_sections[] = {
+struct razor_set_section_index {
+	const char *name;
+	uint32_t offset;
+};
+
+struct razor_set_section_index razor_sections[] = {
 	{ RAZOR_STRING_POOL,	offsetof(struct razor_set, string_pool) },
 	{ RAZOR_PACKAGES,	offsetof(struct razor_set, packages) },
 	{ RAZOR_PROPERTIES,	offsetof(struct razor_set, properties) },
@@ -57,13 +62,13 @@ struct razor_set_section razor_sections[] = {
 	{ RAZOR_PROPERTY_POOL,	offsetof(struct razor_set, property_pool) },
 };
 
-struct razor_set_section razor_files_sections[] = {
+struct razor_set_section_index razor_files_sections[] = {
 	{ RAZOR_FILES,			offsetof(struct razor_set, files) },
 	{ RAZOR_FILE_POOL,		offsetof(struct razor_set, file_pool) },
 	{ RAZOR_FILE_STRING_POOL,	offsetof(struct razor_set, file_string_pool) },
 };
 
-struct razor_set_section razor_details_sections[] = {
+struct razor_set_section_index razor_details_sections[] = {
 	{ RAZOR_DETAILS_STRING_POOL,	offsetof(struct razor_set, details_string_pool) },
 };
 
@@ -87,120 +92,95 @@ razor_set_create(void)
 	return set;
 }
 
-RAZOR_EXPORT struct razor_set *
-razor_set_open(const char *filename)
+static int
+razor_set_bind_sections(struct razor_set *set,
+			struct razor_set_header **header,
+			size_t *header_size,
+			struct razor_set_section_index section_index[],
+			int section_index_size,
+			const char *filename)
 {
-	struct razor_set *set;
-	struct razor_set_section *s;
+	struct razor_set_section *s, *sections;
 	struct stat stat;
 	struct array *array;
-	int fd;
+	const char *pool;
+	int fd, i;
 
-	set = zalloc(sizeof *set);
 	fd = open(filename, O_RDONLY);
 	if (fstat(fd, &stat) < 0)
-		return NULL;
-	set->header = mmap(NULL, stat.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
-	if (set->header == MAP_FAILED) {
-		free(set);
-		return NULL;
-	}
-
-	for (s = set->header->sections; ~s->type; s++) {
-		if (s->type >= ARRAY_SIZE(razor_sections))
-			continue;
-		if (s->type != razor_sections[s->type].type)
+		return -1;
+	*header = mmap(NULL, stat.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
+	if (*header == MAP_FAILED)
+		return -1;
+	*header_size = stat.st_size;
+
+	sections = (void *) *header + sizeof **header;
+	pool = (void *) sections + (*header)->num_sections * sizeof *sections;
+
+	for (i = 0; i < (*header)->num_sections; i++) {
+		int j;
+		s = sections + i;
+		for (j = 0; j < section_index_size; j++)
+			if (!strcmp(section_index[j].name,
+				    &pool[s->name]))
+				break;
+		if (j == section_index_size)
 			continue;
-		array = (void *) set + razor_sections[s->type].offset;
-		array->data = (void *) set->header + s->offset;
+		array = (void *) set + section_index[j].offset;
+		array->data = (void *) *header + s->offset;
 		array->size = s->size;
 		array->alloc = s->size;
 	}
 	close(fd);
 
-	return set;
+	return 0;
 }
 
-RAZOR_EXPORT int
-razor_set_open_details(struct razor_set *set, const char *filename)
+RAZOR_EXPORT struct razor_set *
+razor_set_open(const char *filename)
 {
-	struct razor_set_section *s;
-	struct stat stat;
-	struct array *array;
-	int fd;
-
-	assert (set != NULL);
-	assert (filename != NULL);
-
-	fd = open(filename, O_RDONLY);
-	if (fstat(fd, &stat) < 0)
-		return -1;
-	set->details_header = mmap(NULL, stat.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
-	if (set->details_header == MAP_FAILED)
-		return -1;
+	struct razor_set *set;
 
-	for (s = set->details_header->sections; ~s->type; s++) {
-		if (s->type >= ARRAY_SIZE(razor_details_sections))
-			continue;
-		if (s->type != razor_details_sections[s->type].type)
-			continue;
-		array = (void *) set + razor_details_sections[s->type].offset;
-		array->data = (void *) set->details_header + s->offset;
-		array->size = s->size;
-		array->alloc = s->size;
+	set = zalloc(sizeof *set);
+	if (razor_set_bind_sections(set, &set->header, &set->header_size,
+				    razor_sections, ARRAY_SIZE(razor_sections),
+				    filename)){
+		free(set);
+		return NULL;
 	}
-	close(fd);
+	return set;
+}
 
-	return 0;
+RAZOR_EXPORT int
+razor_set_open_details(struct razor_set *set, const char *filename)
+{
+	return razor_set_bind_sections(set, &set->details_header,
+				       &set->details_header_size,
+				       razor_details_sections,
+				       ARRAY_SIZE(razor_details_sections),
+				       filename);
 }
 
 RAZOR_EXPORT int
 razor_set_open_files(struct razor_set *set, const char *filename)
 {
-	struct razor_set_section *s;
-	struct stat stat;
-	struct array *array;
-	int fd;
-
-	assert (set != NULL);
-	assert (filename != NULL);
-
-	fd = open(filename, O_RDONLY);
-	if (fstat(fd, &stat) < 0)
-		return -1;
-	set->files_header = mmap(NULL, stat.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
-	if (set->files_header == MAP_FAILED)
-		return -1;
-
-	for (s = set->files_header->sections; ~s->type; s++) {
-		if (s->type >= ARRAY_SIZE(razor_files_sections))
-			continue;
-		if (s->type != razor_files_sections[s->type].type)
-			continue;
-		array = (void *) set + razor_files_sections[s->type].offset;
-		array->data = (void *) set->files_header + s->offset;
-		array->size = s->size;
-		array->alloc = s->size;
-	}
-	close(fd);
-
-	return 0;
+	return razor_set_bind_sections(set, &set->files_header,
+				       &set->files_header_size,
+				       razor_files_sections,
+				       ARRAY_SIZE(razor_files_sections),
+				       filename);
 }
 
 RAZOR_EXPORT void
 razor_set_destroy(struct razor_set *set)
 {
-	unsigned int size;
 	struct array *a;
 	int i;
 
 	assert (set != NULL);
 
 	if (set->header) {
-		for (i = 0; set->header->sections[i].type; i++)
-			;
-		size = set->header->sections[i].type;
-		munmap(set->header, size);
+		munmap(set->header, set->header_size);
 	} else {
 		for (i = 0; i < ARRAY_SIZE(razor_sections); i++) {
 			a = (void *) set + razor_sections[i].offset;
@@ -209,10 +189,7 @@ razor_set_destroy(struct razor_set *set)
 	}
 
 	if (set->details_header) {
-		for (i = 0; set->details_header->sections[i].type; i++)
-			;
-		size = set->details_header->sections[i].type;
-		munmap(set->details_header, size);
+		munmap(set->details_header, set->details_header_size);
 	} else {
 		for (i = 0; i < ARRAY_SIZE(razor_details_sections); i++) {
 			a = (void *) set + razor_details_sections[i].offset;
@@ -221,10 +198,7 @@ razor_set_destroy(struct razor_set *set)
 	}
 
 	if (set->files_header) {
-		for (i = 0; set->files_header->sections[i].type; i++)
-			;
-		size = set->files_header->sections[i].type;
-		munmap(set->files_header, size);
+		munmap(set->files_header, set->files_header_size);
 	} else {
 		for (i = 0; i < ARRAY_SIZE(razor_files_sections); i++) {
 			a = (void *) set + razor_files_sections[i].offset;
@@ -236,45 +210,50 @@ razor_set_destroy(struct razor_set *set)
 }
 
 static int
-razor_set_write_sections_to_fd(struct razor_set *set, int fd, int magic,
-			       struct razor_set_section *sections,
+razor_set_write_sections_to_fd(struct razor_set *set, int fd,
+			       struct razor_set_section_index *sections,
 			       size_t array_size)
 {
-	char data[4096];
-	struct razor_set_header *header = (struct razor_set_header *) data;
-	struct array *a;
+	struct razor_set_header header;
+	struct razor_set_section *out_sections =
+		malloc(array_size * sizeof *out_sections);
+	struct hashtable table;
+	struct array *a, pool;
 	uint32_t offset;
 	int i;
 
-	memset(data, 0, sizeof data);
-	header->magic = magic;
-	header->version = RAZOR_VERSION;
-	offset = sizeof data;
+	header.magic = RAZOR_MAGIC;
+	header.version = RAZOR_VERSION;
+	header.num_sections = array_size;
+	offset = sizeof header + array_size * sizeof *out_sections;
+
+	array_init(&pool);
+	hashtable_init(&table, &pool);
+
+	for (i = 0; i < array_size; i++)
+		out_sections[i].name =
+			hashtable_tokenize(&table, sections[i].name);
+
+	offset += pool.size;
 
 	for (i = 0; i < array_size; i++) {
-		if (sections[i].type != i)
-			continue;
 		a = (void *) set + sections[i].offset;
-		header->sections[i].type = i;
-		header->sections[i].offset = offset;
-		header->sections[i].size = a->size;
-		offset += ALIGN(a->size, 4096);
+		out_sections[i].offset = offset;
+		out_sections[i].size = a->size;
+		offset += a->size;
 	}
 
-	header->sections[i].type = ~0;
-	header->sections[i].offset = 0;
-	header->sections[i].size = 0;
+	razor_write(fd, &header, sizeof header);
+	razor_write(fd, out_sections, array_size * sizeof *out_sections);
+	razor_write(fd, pool.data, pool.size);
 
-	razor_write(fd, data, sizeof data);
-	memset(data, 0, sizeof data);
 	for (i = 0; i < array_size; i++) {
-		if (sections[i].type != i)
-			continue;
 		a = (void *) set + sections[i].offset;
 		razor_write(fd, a->data, a->size);
-		razor_write(fd, data, ALIGN(a->size, 4096) - a->size);
 	}
 
+	free(out_sections);
+
 	return 0;
 }
 
@@ -284,16 +263,16 @@ razor_set_write_to_fd(struct razor_set *set, int fd,
 {
 	switch (type) {
 	case RAZOR_REPO_FILE_MAIN:
-		return razor_set_write_sections_to_fd(set, fd, RAZOR_MAGIC,
+		return razor_set_write_sections_to_fd(set, fd,
 						      razor_sections,
 						      ARRAY_SIZE(razor_sections));
 
 	case RAZOR_REPO_FILE_DETAILS:
-		return razor_set_write_sections_to_fd(set, fd, RAZOR_DETAILS_MAGIC,
+		return razor_set_write_sections_to_fd(set, fd,
 						      razor_details_sections,
 						      ARRAY_SIZE(razor_details_sections));
 	case RAZOR_REPO_FILE_FILES:
-		return razor_set_write_sections_to_fd(set, fd, RAZOR_FILES_MAGIC,
+		return razor_set_write_sections_to_fd(set, fd,
 						      razor_files_sections,
 						      ARRAY_SIZE(razor_files_sections));
 	default: