[Toybox] [PATCH v2] Generate all build files with single program.
Georgi Chorbadzhiyski
gf at unixsol.org
Mon Mar 19 05:32:59 PDT 2012
This patch removes all external tools from the build system. Now
everything in generated/ directory is build with single program.
The effect is that the build no longer requires sed, tr, sort and
python (for rebuilding the help texts). There is no need to ship
generated/help.h since it can be rebuild on the host without python.
There is also a nice speedup. On my machine building allyesconfig
is ~1.8 seconds faster, it goes down from 11.3s to 0m9.5s.
---
v2 changelog:
In this version input files are sorted before processing and also
entries in newtoys.h are sorted before saving them.
Makefile | 7 +-
scripts/config2help.py | 54 -------
scripts/config2help.sh | 54 -------
scripts/genconf.c | 408 ++++++++++++++++++++++++++++++++++++++++++++++++
scripts/genconfig.sh | 10 +-
scripts/make.sh | 88 +----------
6 files changed, 423 insertions(+), 198 deletions(-)
delete mode 100755 scripts/config2help.py
delete mode 100755 scripts/config2help.sh
create mode 100644 scripts/genconf.c
diff --git a/Makefile b/Makefile
index 7079f82..4aec0ff 100644
--- a/Makefile
+++ b/Makefile
@@ -17,6 +17,9 @@ generated/Config.in: toys/*.c scripts/genconfig.sh
HOSTCC?=cc
+.config:
+ $(MAKE) defconfig
+
# Development targets
baseline: toybox_unstripped
@cp toybox_unstripped toybox_old
@@ -42,10 +45,10 @@ uninstall:
clean::
rm -rf toybox toybox_unstripped generated/config.h generated/Config.in \
generated/newtoys.h generated/globals.h instlist testdir \
- generated/Config.probed
+ generated/Config.probed generated/help.h generated/build_files
distclean: clean
- rm -f toybox_old .config* generated/help.h
+ rm -f toybox_old .config* generated/genconf
test: tests
diff --git a/scripts/config2help.py b/scripts/config2help.py
deleted file mode 100755
index 2573d08..0000000
--- a/scripts/config2help.py
+++ /dev/null
@@ -1,54 +0,0 @@
-#!/usr/bin/python
-
-import os,sys
-
-def zapquotes(str):
- if str[0]=='"': str = str[1:str.rfind('"')]
- return str
-
-def escapequotes(str):
- return str.strip().replace("\\","\\\\").replace('"','\\"')
-
-helplen = morelines = 0
-out = sys.stdout
-
-def readfile(filename):
- global helplen, morelines
- #sys.stderr.write("Reading %s\n" % filename)
- try:
- lines = open(filename).read().split("\n")
- except IOError:
- sys.stderr.write("File %s missing\n" % filename)
- return
- config = None
- description = None
- for i in lines:
- if helplen:
- i = i.expandtabs()
- if not len(i) or i[:helplen].isspace():
- if morelines: out.write('\\n')
- morelines = 1
- out.write(escapequotes(i))
- continue
- else:
- helplen = morelines = 0
- out.write('"\n')
-
- words = i.strip().split(None,1)
- if not len(words): continue
-
- if words[0] in ("config", "menuconfig"):
- config = words[1]
- description = ""
- elif words[0] in ("bool", "boolean", "tristate", "string", "hex", "int"):
- if len(words)>1: description = zapquotes(words[1])
- elif words[0]=="prompt":
- description = htmlescape(zapquotes(words[1]))
- elif words[0] in ("help", "---help---"):
- out.write('#define help_%s "' % config.lower())
- helplen = len(i[:i.find(words[0])].expandtabs())
- elif words[0] == "source": readfile(zapquotes(words[1]))
- elif words[0] in ("default","depends", "select", "if", "endif", "#", "comment", "menu", "endmenu"): pass
-
-readfile(sys.argv[1])
-if helplen: out.write('"\n')
diff --git a/scripts/config2help.sh b/scripts/config2help.sh
deleted file mode 100755
index 63d6c6f..0000000
--- a/scripts/config2help.sh
+++ /dev/null
@@ -1,54 +0,0 @@
-#!/bin/bash -e
-
-function firstmajor() {
- declare -i n j=$1
- test $j -gt 0 || return -1
- for j in $@; do
- if [ $j -gt $1 ]; then
- echo $j
- return 0
- fi
- done
- return 1
-}; export -f firstmajor
-
-function print_h() {
- declare -i i c=$2 s=$3 e=$4
- local str="$(echo "$1" | head -n$c | tail -n1 | sed -e "s,config[\t ]*,,")"
- echo -n "#define help_"$str" \"" | tr [A-Z] [a-z]
- echo -n "$1\\n" | head -n$e | tail -n$[e-s+1] | sed -e "s,\$,\r," | tr \\n\\r n\\
- echo \"
-}; export -f print_h
-
-file="$1"
-if test "$0" != "bash"; then
- if test -r "$file"; then
-# echo "$file..." >&2
- filetxt="$(sed -e "s,^[\t ]*,," -e "s,\([\"\\\\]\),\\\\\\1,g" "$file")"
- helps=$(echo "$filetxt" | egrep -ne "^help *" -e "^---help--- *" | cut -d\: -f1)
- configs=$(echo "$filetxt" | egrep -ne "^config *" | cut -d\: -f1)
- endmenus=$(echo "$filetxt" | egrep -ne "^endmenu *" | cut -d\: -f1)
- let last=$(echo "$filetxt" | wc -l)+2
-
- declare -i i c s e
- for i in $configs; do
-# echo -n "c:$i" >&2
- s=$(firstmajor $i $helps)
- test $s -gt 0
- e=$(firstmajor $s $configs || firstmajor $s $endmenus $last)
- let s++ e-=2
- test $e -ge $s
-# echo " s:$s e:$e" >&2
- print_h "$filetxt" $i $s $e
- done
- for fle in $(cat "$file" | egrep -e "^[ \t]*source " | sed -e "s,^[ \t]*source *\(.*\),\\1,"); do
- $0 $fle
- done
- else
- echo
- echo "USAGE EXAMPLE: $(basename $0) Config.in > generated/help.h"
- echo
- false
- fi | sed -e "s,\\\\n\\\\n\"$,\\\\n\","
-fi
-
diff --git a/scripts/genconf.c b/scripts/genconf.c
new file mode 100644
index 0000000..e93954a
--- /dev/null
+++ b/scripts/genconf.c
@@ -0,0 +1,408 @@
+/* vi: set ts=4 :*/
+/*
+ * Generate toybox generated/{Config.in,config.h,globals.h,newtoys.h,help.h}
+ *
+ * Copyright 2012 Georgi Chorbadzhiyski <georgi at unixsol.org>
+ */
+#include <stdio.h>
+#include <stdarg.h>
+#include <stdlib.h>
+#include <string.h>
+#include <ctype.h>
+#include <unistd.h>
+#include <errno.h>
+
+struct strbuf {
+ unsigned int len;
+ unsigned int nalloc;
+ char *data;
+};
+
+struct linebuf {
+ unsigned int num;
+ unsigned int nalloc;
+ char **data;
+};
+
+struct strbuf config_h;
+struct strbuf config_in;
+struct strbuf help_h;
+struct strbuf globals_h;
+struct strbuf globals_union;
+struct strbuf build_files;
+
+struct linebuf input_files;
+struct linebuf newtoys_h;
+
+int in_config;
+int in_help;
+
+int i;
+char *line;
+size_t linelen;
+char tmp[1024];
+char *conf, *pos;
+FILE *f;
+
+// Library functions
+static FILE *xfopen(char *filename, char *mode);
+
+static void linebuf_add(struct linebuf *l, char *text);
+static void linebuf_sort(struct linebuf *l);
+static void linebuf_free(struct linebuf *l);
+
+static void strbuf_add(struct strbuf *s, char *text);
+static void strbuf_printf(struct strbuf *s, const char *fmt, ...);
+static void strbuf_add_escaped(struct strbuf *s, char *text);
+static void strbuf_dump(struct strbuf *s, char *filename, char *mode);
+static void strbuf_free(struct strbuf *s);
+
+/*
+ * Parse KConfig Config.in style file and add help entries in help_h
+ */
+static void parse_help(char *filename) {
+ int i, nitems = 0;
+ char *config_name = NULL;
+
+ f = xfopen(filename, "r");
+ while (getline(&line, &linelen, f) > 0) {
+ if (line[0] == '#')
+ continue;
+ if (strncmp(line, "config ", 7) == 0) {
+ in_config = 1;
+ in_help = 0;
+ free(config_name);
+ config_name = strdup(line + 7);
+ int config_len = strlen(config_name);
+ for (i = 0; i < config_len; i++) {
+ config_name[i] = tolower((unsigned char)config_name[i]);
+ if (config_name[i] == '\n')
+ config_name[i] = '\0';
+ }
+ // Close already started help line
+ if (nitems)
+ strbuf_add(&help_h, "\"\n");
+ continue;
+ }
+ if (strncmp(line, "endmenu", 7) == 0) {
+ in_config = 0;
+ in_help = 0;
+ }
+ if (!in_config)
+ continue;
+
+ if (!in_help) {
+ if (strncmp(line, "\thelp", 5) == 0 || strncmp(line, " help", 8) == 0) {
+ in_help = 1;
+ nitems++;
+ strbuf_printf(&help_h, "#define help_%s \"", config_name);
+ }
+ continue;
+ } else {
+ char *tmpline = line;
+ while (isspace(*tmpline))
+ tmpline++;
+ strbuf_add_escaped(&help_h, tmpline);
+ continue;
+ }
+ }
+ strbuf_add(&help_h, "\"\n");
+
+ fclose(f);
+
+ free(config_name);
+}
+
+/*
+ * Parse Config.in and generate/Config.in and create generated/{help.h}
+ * Must be called after generated/Config.in is created from parse_toy_files().
+ */
+static void generate_config_in() {
+ parse_help("Config.in");
+ parse_help("generated/Config.in");
+ strbuf_dump(&help_h, "generated/help.h", "w");
+}
+
+/*
+ * Parse input files and create generated/{Config.in,globals.h,newtoys.h}
+ */
+static void parse_toy_files() {
+ int is_toy = 0, in_globals = 0;
+
+ strbuf_add(&globals_union, "extern union global_union {\n");
+
+ // Parse toys
+ for (i = 0; i < input_files.num; i++) {
+ char *filename = input_files.data[i];
+ f = xfopen(filename, "r");
+
+ char *toy = strdup(filename);
+ char *toyname = toy;
+ pos = strrchr(toyname, '/');
+ if (pos)
+ toyname = pos + 1; // Remove path
+ pos = strstr(toyname, ".c");
+ if (pos) // Remove extension
+ pos[0] = '\0';
+
+ strbuf_printf(&globals_h, "// %s\n\n", filename);
+
+ while (getline(&line, &linelen, f) > 0) {
+ // Parse toys
+ if (strncmp(line, "USE_", 4) == 0) {
+ is_toy = 1;
+ linebuf_add(&newtoys_h, line);
+ continue;
+ }
+ if (!is_toy)
+ continue;
+
+ // Parse config
+ if (!in_config) {
+ if (strncmp(line, "config ", 7) == 0) {
+ in_config = 1;
+ strbuf_printf(&config_in, "# %s\n", filename);
+ strbuf_add(&config_in, line);
+ continue;
+ }
+ } else {
+ if (strncmp(line, "*/", 2) == 0) {
+ strbuf_add(&config_in, "\n");
+ in_config = 0;
+ continue;
+ } else {
+ strbuf_add(&config_in, line);
+ }
+ }
+
+ // Parse globals
+ if (!in_globals) {
+ if (strncmp(line, "DEFINE_GLOBALS(", 15) == 0) {
+ strbuf_printf(&globals_h, "struct %s_data {\n", toyname);
+ strbuf_printf(&globals_union, "\tstruct %s_data %s;\n", toyname, toyname);
+ in_globals = 1;
+ continue;
+ }
+ } else {
+ if (strncmp(line, ")", 1) == 0) {
+ strbuf_add(&globals_h, "};\n");
+ in_globals = 0;
+ break;
+ } else {
+ strbuf_add(&globals_h, line);
+ }
+ }
+ }
+
+ free(toy);
+ fclose(f);
+ }
+
+ strbuf_add(&globals_union, "} this;\n");
+
+ strbuf_dump(&config_in, "generated/Config.in", "w");
+ strbuf_dump(&globals_h, "generated/globals.h", "w");
+ strbuf_dump(&globals_union, "generated/globals.h", "a");
+
+ // Write newtoys.h
+ linebuf_sort(&newtoys_h);
+ f = xfopen("generated/newtoys.h", "w");
+ const char *first_toy = "NEWTOY(toybox, NULL, 0)\n";
+ fwrite(first_toy, strlen(first_toy), 1, f);
+ for (i = 0; i < newtoys_h.num; i++) {
+ fwrite(newtoys_h.data[i], strlen(newtoys_h.data[i]), 1, f);
+ }
+ fclose(f);
+}
+
+/*
+ * Parse .config file and generate generated/config.h and generate/build_files
+ */
+static void parse_config() {
+ // Configu do not exists, create dummy .config
+ if (!access(".config", R_OK) == 0) {
+ f = xfopen(".config", "w");
+ fclose(f);
+ return;
+ }
+
+ // Parse .config
+ f = xfopen(".config", "r");
+ while (getline(&line, &linelen, f) > 0) {
+ if (strncmp(line, "CONFIG_", 7) == 0) {
+ // Config is ON
+ conf = line + 7;
+ pos = strchr(conf, '=');
+ if (pos) pos[0] = '\0';
+ strbuf_printf(&config_h, "#define CFG_%s 1\n", conf);
+ strbuf_printf(&config_h, "#define USE_%s(...) __VA_ARGS__\n", conf);
+ if (strncmp(conf, "TOYBOX_", 7) != 0) {
+ // Generate build_files
+ int config_len = strlen(conf);
+ for (i = 0; i < config_len; i++) {
+ conf[i] = tolower((unsigned char)conf[i]);
+ if (conf[i] == '\n' || conf[i] == ' ' || conf[i] == '=')
+ conf[i] = '\0';
+ }
+ snprintf(tmp, sizeof(tmp) - 1, "toys/%s.c", conf);
+ tmp[sizeof(tmp) - 1] = '\0';
+ if (access(tmp, R_OK) == 0)
+ strbuf_printf(&build_files, "%s\n", tmp);
+ }
+ continue;
+ }
+ if (strncmp(line, "# CONFIG_", 9) == 0) {
+ // Config is OFF
+ conf = line + 9;
+ pos = strchr(conf, ' ');
+ if (pos) pos[0] = '\0';
+ strbuf_printf(&config_h, "#define CFG_%s 0\n", conf);
+ strbuf_printf(&config_h, "#define USE_%s(...)\n", conf);
+ continue;
+ }
+ }
+ fclose(f);
+ strbuf_dump(&config_h, "generated/config.h", "w");
+ strbuf_dump(&build_files, "generated/build_files", "w");
+}
+
+/*
+ * MAIN
+ */
+int main(int argc, char **argv) {
+ if (argc < 2) {
+ fprintf(stderr, "Usage: %s toys/*.c\n", argv[0]);
+ exit(1);
+ }
+
+ for (i = 1; i < argc; i++) {
+ linebuf_add(&input_files, argv[i]);
+ }
+ linebuf_sort(&input_files);
+
+ parse_toy_files();
+ generate_config_in();
+ parse_config();
+
+ linebuf_free(&input_files);
+ linebuf_free(&newtoys_h);
+
+ strbuf_free(&config_h);
+ strbuf_free(&config_in);
+ strbuf_free(&help_h);
+ strbuf_free(&globals_h);
+ strbuf_free(&globals_union);
+ strbuf_free(&build_files);
+
+ free(line);
+
+ return 0;
+}
+
+/*
+ * Library functions
+ */
+static FILE *xfopen(char *filename, char *mode) {
+ FILE *fh = fopen(filename, mode);
+ if (!fh) {
+ fprintf(stderr, "fopen(%s, %s): %s\n", filename, mode, strerror(errno));
+ exit(-1);
+ }
+ return fh;
+}
+
+static void linebuf_add(struct linebuf *l, char *text) {
+ int len = strlen(text);
+ if (!len)
+ return;
+ if (!l->data) {
+ l->nalloc = 1024;
+ l->data = calloc(l->nalloc, sizeof(char *));
+ }
+ if (l->num + 1 >= l->nalloc) {
+ l->nalloc *= 2;
+ l->data = realloc(l->data, l->nalloc * sizeof(char *));
+ if (!l->data) {
+ fprintf(stderr, "realloc(%d): %s\n", l->nalloc * sizeof(char *), strerror(errno));
+ exit(-1);
+ }
+ }
+ l->data[l->num] = strdup(text);
+ if (l->data[l->num])
+ l->num++;
+}
+
+static int cmp_string(const void *p1, const void *p2) {
+ return strcmp(* (char * const *) p1, * (char * const *) p2);
+}
+
+static void linebuf_sort(struct linebuf *l) {
+ qsort(l->data, l->num, sizeof(char *), cmp_string);
+}
+
+static void linebuf_free(struct linebuf *l) {
+ free(l->data);
+}
+
+static void strbuf_add(struct strbuf *s, char *text) {
+ int len = strlen(text);
+ if (!len)
+ return;
+ if (!s->data) {
+ s->nalloc = 1024;
+ s->data = calloc(1, s->nalloc);
+ }
+ s->len += len;
+ if (s->len >= s->nalloc) {
+ s->nalloc += s->len + 1;
+ s->data = realloc(s->data, s->nalloc);
+ if (!s->data) {
+ fprintf(stderr, "realloc(%d): %s\n", s->nalloc, strerror(errno));
+ exit(-1);
+ }
+ }
+ strcat(s->data, text);
+}
+
+static void strbuf_printf(struct strbuf *s, const char *fmt, ...) {
+ char line[4096];
+
+ va_list args;
+ va_start(args, fmt);
+ vsnprintf(line, sizeof(line) - 1, fmt, args);
+ va_end(args);
+
+ line[sizeof(line) - 1] = '\0';
+ strbuf_add(s, line);
+}
+
+static void strbuf_add_escaped(struct strbuf *s, char *text) {
+ int len = strlen(text), i;
+ if (len == 0)
+ strbuf_add(s, "\\n");
+ for (i = 0; i < len; i++) {
+ char c[2] = { text[i], 0 };
+ switch (text[i]) {
+ case '\n': strbuf_add(s, "\\n"); break;
+ case '\t': strbuf_add(s, " "); break;
+ case '\\': strbuf_add(s, "\\\\"); break;
+ case '"' : strbuf_add(s, "\\\""); break;
+ default : strbuf_add(s, c); break;
+ }
+ }
+}
+
+static void strbuf_dump(struct strbuf *s, char *filename, char *mode) {
+ FILE *fh;
+ if (!s->data) {
+ unlink(filename);
+ return;
+ }
+ fh = xfopen(filename, mode);
+ fwrite(s->data, s->len, 1, fh);
+ fclose(fh);
+}
+
+static void strbuf_free(struct strbuf *s) {
+ free(s->data);
+}
diff --git a/scripts/genconfig.sh b/scripts/genconfig.sh
index f677954..c562e37 100755
--- a/scripts/genconfig.sh
+++ b/scripts/genconfig.sh
@@ -34,4 +34,12 @@ genconfig()
}
probeconfig > generated/Config.probed || rm generated/Config.probed
-genconfig > generated/Config.in || rm generated/Config.in
+
+if [ ! -x generated/genconf ]
+then
+ echo "Building generated/genconf"
+ $HOSTCC -ggdb -Wall -Wextra -Wno-sign-compare -I . -o generated/genconf scripts/genconf.c || exit 1
+fi
+
+echo "Extract configuration information from toys/*.c files..."
+generated/genconf toys/*.c || exit 1
diff --git a/scripts/make.sh b/scripts/make.sh
index f5bf0fd..e9afa68 100755
--- a/scripts/make.sh
+++ b/scripts/make.sh
@@ -4,95 +4,9 @@
source ./configure
-echo "Extract configuration information from toys/*.c files..."
scripts/genconfig.sh
-echo "Generate headers from toys/*.c..."
-
-# Create a list of all the applets toybox can provide. Note that the first
-# entry is out of order on purpose (the toybox multiplexer applet must be the
-# first element of the array). The rest must be sorted in alphabetical order
-# for fast binary search.
-
-function newtoys()
-{
- for i in toys/*.c
- do
- sed -n -e '1,/^config [A-Z]/s/^USE_/&/p' $i || exit 1
- done
-}
-echo "NEWTOY(toybox, NULL, 0)" > generated/newtoys.h
-newtoys | sed 's/\(.*TOY(\)\([^,]*\),\(.*\)/\2 \1\2,\3/' | sort -k 1,1 \
- | sed 's/[^ ]* //' >> generated/newtoys.h
-
-# Extract global structure definitions from toys/*.c
-
-function getglobals()
-{
- for i in toys/*.c
- do
- NAME="$(echo $i | sed 's at toys/\(.*\)\.c@\1@')"
-
- echo -e "// $i\n"
- sed -n -e '/^DEFINE_GLOBALS(/,/^)/b got;b;:got' \
- -e 's/^DEFINE_GLOBALS(/struct '"$NAME"'_data {/' \
- -e 's/^)/};/' -e 'p' $i
- done
-}
-
-GLOBSTRUCT="$(getglobals)"
-(
- echo "$GLOBSTRUCT"
- echo
- echo "extern union global_union {"
- echo "$GLOBSTRUCT" | sed -n 's/struct \(.*\)_data {/ struct \1_data \1;/p'
- echo "} this;"
-) > generated/globals.h
-
-# Only recreate generated/help.h if python is installed
-if [ ! -z "$(which python)" ] && [ ! -z "$(grep 'CONFIG_HELP=y' .config)" ]
-then
- echo "Extract help text from Config.in."
- scripts/config2help.py Config.in > generated/help.h || exit 1
-fi
-
-echo "Make generated/config.h from .config."
-
-# This long and roundabout sed invocation is to make old versions of sed happy.
-# New ones have '\n' so can replace one line with two without all the branches
-# and tedious mucking about with hold space.
-
-sed -n \
- -e 's/^# CONFIG_\(.*\) is not set.*/\1/' \
- -e 't notset' \
- -e 's/^CONFIG_\(.*\)=y.*/\1/' \
- -e 't isset' \
- -e 's/^CONFIG_\([^=]*\)=\(.*\)/#define CFG_\1 \2/p' \
- -e 'd' \
- -e ':notset' \
- -e 'h' \
- -e 's/.*/#define CFG_& 0/p' \
- -e 'g' \
- -e 's/.*/#define USE_&(...)/p' \
- -e 'd' \
- -e ':isset' \
- -e 'h' \
- -e 's/.*/#define CFG_& 1/p' \
- -e 'g' \
- -e 's/.*/#define USE_&(...) __VA_ARGS__/p' \
- .config > generated/config.h || exit 1
-
-# Extract a list of toys/*.c files to compile from the data in ".config" with
-# sed, sort, and tr:
-
-# 1) Grab the XXX part of all CONFIG_XXX entries, removing everything after the
-# second underline
-# 2) Sort the list, keeping only one of each entry.
-# 3) Convert to lower case.
-# 4) Remove toybox itself from the list (as that indicates global symbols).
-# 5) Add "toys/" prefix and ".c" suffix.
-
-TOYFILES=$(cat .config | sed -nre 's/^CONFIG_(.*)=y/\1/;t skip;b;:skip;s/_.*//;p' | sort -u | tr A-Z a-z | grep -v '^toybox$' | sed 's@\(.*\)@toys/\1.c@' )
+TOYFILES=$(cat generated/build_files 2>/dev/null)
echo "Compile toybox..."
--
1.7.5.1
More information about the Toybox
mailing list