[Toybox] [PATCH] Generate all build files with single program.
Georgi Chorbadzhiyski
gf at unixsol.org
Fri Mar 16 15:05:06 PDT 2012
This patch removes all external tools from the build system. Now
everything in generated/ directory is build with single program.
The effect is that the build no longer requires sed, tr, sort and
python (for rebuilding the help texts). There is no need to ship
generated/help.h since it can be rebuild on the host without python.
There is also a nice speedup. On my machine building allyesconfig
is ~1.8 seconds faster, it goes down from 11.3s to 0m9.5s.
---
Makefile | 7 +-
scripts/config2help.py | 54 ---------
scripts/config2help.sh | 54 ---------
scripts/genconf.c | 309 ++++++++++++++++++++++++++++++++++++++++++++++++
scripts/genconfig.sh | 10 ++-
scripts/make.sh | 88 +--------------
6 files changed, 324 insertions(+), 198 deletions(-)
delete mode 100755 scripts/config2help.py
delete mode 100755 scripts/config2help.sh
create mode 100644 scripts/genconf.c
diff --git a/Makefile b/Makefile
index 7079f82..4aec0ff 100644
--- a/Makefile
+++ b/Makefile
@@ -17,6 +17,9 @@ generated/Config.in: toys/*.c scripts/genconfig.sh
HOSTCC?=cc
+.config:
+ $(MAKE) defconfig
+
# Development targets
baseline: toybox_unstripped
@cp toybox_unstripped toybox_old
@@ -42,10 +45,10 @@ uninstall:
clean::
rm -rf toybox toybox_unstripped generated/config.h generated/Config.in \
generated/newtoys.h generated/globals.h instlist testdir \
- generated/Config.probed
+ generated/Config.probed generated/help.h generated/build_files
distclean: clean
- rm -f toybox_old .config* generated/help.h
+ rm -f toybox_old .config* generated/genconf
test: tests
diff --git a/scripts/config2help.py b/scripts/config2help.py
deleted file mode 100755
index 2573d08..0000000
--- a/scripts/config2help.py
+++ /dev/null
@@ -1,54 +0,0 @@
-#!/usr/bin/python
-
-import os,sys
-
-def zapquotes(str):
- if str[0]=='"': str = str[1:str.rfind('"')]
- return str
-
-def escapequotes(str):
- return str.strip().replace("\\","\\\\").replace('"','\\"')
-
-helplen = morelines = 0
-out = sys.stdout
-
-def readfile(filename):
- global helplen, morelines
- #sys.stderr.write("Reading %s\n" % filename)
- try:
- lines = open(filename).read().split("\n")
- except IOError:
- sys.stderr.write("File %s missing\n" % filename)
- return
- config = None
- description = None
- for i in lines:
- if helplen:
- i = i.expandtabs()
- if not len(i) or i[:helplen].isspace():
- if morelines: out.write('\\n')
- morelines = 1
- out.write(escapequotes(i))
- continue
- else:
- helplen = morelines = 0
- out.write('"\n')
-
- words = i.strip().split(None,1)
- if not len(words): continue
-
- if words[0] in ("config", "menuconfig"):
- config = words[1]
- description = ""
- elif words[0] in ("bool", "boolean", "tristate", "string", "hex", "int"):
- if len(words)>1: description = zapquotes(words[1])
- elif words[0]=="prompt":
- description = htmlescape(zapquotes(words[1]))
- elif words[0] in ("help", "---help---"):
- out.write('#define help_%s "' % config.lower())
- helplen = len(i[:i.find(words[0])].expandtabs())
- elif words[0] == "source": readfile(zapquotes(words[1]))
- elif words[0] in ("default","depends", "select", "if", "endif", "#", "comment", "menu", "endmenu"): pass
-
-readfile(sys.argv[1])
-if helplen: out.write('"\n')
diff --git a/scripts/config2help.sh b/scripts/config2help.sh
deleted file mode 100755
index 63d6c6f..0000000
--- a/scripts/config2help.sh
+++ /dev/null
@@ -1,54 +0,0 @@
-#!/bin/bash -e
-
-function firstmajor() {
- declare -i n j=$1
- test $j -gt 0 || return -1
- for j in $@; do
- if [ $j -gt $1 ]; then
- echo $j
- return 0
- fi
- done
- return 1
-}; export -f firstmajor
-
-function print_h() {
- declare -i i c=$2 s=$3 e=$4
- local str="$(echo "$1" | head -n$c | tail -n1 | sed -e "s,config[\t ]*,,")"
- echo -n "#define help_"$str" \"" | tr [A-Z] [a-z]
- echo -n "$1\\n" | head -n$e | tail -n$[e-s+1] | sed -e "s,\$,\r," | tr \\n\\r n\\
- echo \"
-}; export -f print_h
-
-file="$1"
-if test "$0" != "bash"; then
- if test -r "$file"; then
-# echo "$file..." >&2
- filetxt="$(sed -e "s,^[\t ]*,," -e "s,\([\"\\\\]\),\\\\\\1,g" "$file")"
- helps=$(echo "$filetxt" | egrep -ne "^help *" -e "^---help--- *" | cut -d\: -f1)
- configs=$(echo "$filetxt" | egrep -ne "^config *" | cut -d\: -f1)
- endmenus=$(echo "$filetxt" | egrep -ne "^endmenu *" | cut -d\: -f1)
- let last=$(echo "$filetxt" | wc -l)+2
-
- declare -i i c s e
- for i in $configs; do
-# echo -n "c:$i" >&2
- s=$(firstmajor $i $helps)
- test $s -gt 0
- e=$(firstmajor $s $configs || firstmajor $s $endmenus $last)
- let s++ e-=2
- test $e -ge $s
-# echo " s:$s e:$e" >&2
- print_h "$filetxt" $i $s $e
- done
- for fle in $(cat "$file" | egrep -e "^[ \t]*source " | sed -e "s,^[ \t]*source *\(.*\),\\1,"); do
- $0 $fle
- done
- else
- echo
- echo "USAGE EXAMPLE: $(basename $0) Config.in > generated/help.h"
- echo
- false
- fi | sed -e "s,\\\\n\\\\n\"$,\\\\n\","
-fi
-
diff --git a/scripts/genconf.c b/scripts/genconf.c
new file mode 100644
index 0000000..8346dee
--- /dev/null
+++ b/scripts/genconf.c
@@ -0,0 +1,309 @@
+/* vi: set ts=4 :*/
+/*
+ * Generate toybox generated/{Config.in,config.h,globals.h,newtoys.h,help.h}
+ *
+ * Copyright 2012 Georgi Chorbadzhiyski <georgi at unixsol.org>
+ */
+#include <stdio.h>
+#include <stdarg.h>
+#include <stdlib.h>
+#include <string.h>
+#include <ctype.h>
+#include <unistd.h>
+#include <errno.h>
+
+struct strbuf {
+ unsigned int len;
+ unsigned int nchars;
+ char *data;
+};
+
+struct strbuf newtoys_h;
+struct strbuf config_h;
+struct strbuf config_in;
+struct strbuf help_h;
+struct strbuf globals_h;
+struct strbuf globals_union;
+struct strbuf build_files;
+
+int is_toy;
+int in_config;
+int in_help;
+int in_globals;
+
+char *line = NULL;
+size_t linelen;
+char *pos, *toyname, *conf;
+int i;
+char tmp[1024];
+
+
+FILE *f;
+
+FILE *xfopen(char *filename, char *mode) {
+ FILE *fh = fopen(filename, mode);
+ if (!fh) {
+ fprintf(stderr, "fopen(%s, %s): %s\n", filename, mode, strerror(errno));
+ exit(-1);
+ }
+ return fh;
+}
+
+void strbuf_add(struct strbuf *s, char *text) {
+ int len = strlen(text);
+ if (!len)
+ return;
+ if (!s->data) {
+ s->nchars = 1024;
+ s->data = calloc(1, s->nchars);
+ }
+ s->len += len;
+ if (s->len >= s->nchars) {
+ s->nchars += s->len + 1;
+ s->data = realloc(s->data, s->nchars);
+ if (!s->data) {
+ fprintf(stderr, "realloc(%d): %s\n", s->nchars, strerror(errno));
+ exit(-1);
+ }
+ }
+ strcat(s->data, text);
+}
+
+__attribute__ ((format(printf, 2, 3)))
+void strbuf_printf(struct strbuf *s, const char *fmt, ...) {
+ char line[4096];
+
+ va_list args;
+ va_start(args, fmt);
+ vsnprintf(line, sizeof(line) - 1, fmt, args);
+ va_end(args);
+
+ line[sizeof(line) - 1] = '\0';
+ strbuf_add(s, line);
+}
+
+void strbuf_add_escaped(struct strbuf *s, char *text) {
+ int len = strlen(text), i;
+ if (len == 0)
+ strbuf_add(s, "\\n");
+ for (i = 0; i < len; i++) {
+ char c[2] = { text[i], 0 };
+ switch (text[i]) {
+ case '\n': strbuf_add(s, "\\n"); break;
+ case '\t': strbuf_add(s, " "); break;
+ case '\\': strbuf_add(s, "\\\\"); break;
+ case '"' : strbuf_add(s, "\\\""); break;
+ default : strbuf_add(s, c); break;
+ }
+ }
+}
+
+void strbuf_dump(struct strbuf *s, char *filename, char *mode) {
+ FILE *fh;
+ if (!s->data) {
+ unlink(filename);
+ return;
+ }
+ fh = xfopen(filename, mode);
+ fwrite(s->data, s->len, 1, fh);
+ fclose(fh);
+ free(s->data);
+}
+
+void parse_help(char *filename) {
+ int i, nitems = 0;
+ char *config_name = NULL;
+
+ f = xfopen(filename, "r");
+ while (getline(&line, &linelen, f) > 0) {
+ if (line[0] == '#')
+ continue;
+ if (strncmp(line, "config ", 7) == 0) {
+ in_config = 1;
+ in_help = 0;
+ free(config_name);
+ config_name = strdup(line + 7);
+ int config_len = strlen(config_name);
+ for (i = 0; i < config_len; i++) {
+ config_name[i] = tolower((unsigned char)config_name[i]);
+ if (config_name[i] == '\n')
+ config_name[i] = '\0';
+ }
+ // Close already started help line
+ if (nitems)
+ strbuf_add(&help_h, "\"\n");
+ continue;
+ }
+ if (strncmp(line, "endmenu", 7) == 0) {
+ in_config = 0;
+ in_help = 0;
+ }
+ if (!in_config)
+ continue;
+
+ if (!in_help) {
+ if (strncmp(line, "\thelp", 5) == 0 || strncmp(line, " help", 8) == 0) {
+ in_help = 1;
+ nitems++;
+ strbuf_printf(&help_h, "#define help_%s \"", config_name);
+ }
+ continue;
+ } else {
+ char *tmpline = line;
+ while (isspace(*tmpline))
+ tmpline++;
+ strbuf_add_escaped(&help_h, tmpline);
+ continue;
+ }
+ }
+ strbuf_add(&help_h, "\"\n");
+
+ fclose(f);
+
+ free(config_name);
+}
+
+void generate_headers(int argc, char **argv) {
+ // Parse toys
+ strbuf_add(&newtoys_h, "NEWTOY(toybox, NULL, 0)\n");
+ strbuf_add(&globals_union, "extern union global_union {\n");
+
+ for (i = 1; i < argc; i++) {
+ f = xfopen(argv[i], "r");
+
+ toyname = argv[i];
+ pos = strrchr(toyname, '/');
+ if (pos)
+ toyname = pos + 1; // Remove path
+ pos = strstr(toyname, ".c");
+ if (pos) // Remove extension
+ pos[0] = '\0';
+
+ strbuf_printf(&globals_h, "// %s.c\n\n", argv[i]);
+
+ while (getline(&line, &linelen, f) > 0) {
+ // Parse toys
+ if (strncmp(line, "USE_", 4) == 0) {
+ is_toy = 1;
+ strbuf_add(&newtoys_h, line);
+ continue;
+ }
+ if (!is_toy)
+ continue;
+
+ // Parse config
+ if (!in_config) {
+ if (strncmp(line, "config ", 7) == 0) {
+ in_config = 1;
+ strbuf_printf(&config_in, "# %s.c\n", argv[i]);
+ strbuf_add(&config_in, line);
+ continue;
+ }
+ } else {
+ if (strncmp(line, "*/", 2) == 0) {
+ strbuf_add(&config_in, "\n");
+ in_config = 0;
+ continue;
+ } else {
+ strbuf_add(&config_in, line);
+ }
+ }
+
+ // Parse globals
+ if (!in_globals) {
+ if (strncmp(line, "DEFINE_GLOBALS(", 15) == 0) {
+ strbuf_printf(&globals_h, "struct %s_data {\n", toyname);
+ strbuf_printf(&globals_union, "\tstruct %s_data %s;\n", toyname, toyname);
+ in_globals = 1;
+ continue;
+ }
+ } else {
+ if (strncmp(line, ")", 1) == 0) {
+ strbuf_add(&globals_h, "};\n");
+ in_globals = 0;
+ break;
+ } else {
+ strbuf_add(&globals_h, line);
+ }
+ }
+ }
+ fclose(f);
+ }
+
+ strbuf_add(&globals_union, "} this;\n");
+
+ strbuf_dump(&config_in, "generated/Config.in", "w");
+ strbuf_dump(&globals_h, "generated/globals.h", "w");
+ strbuf_dump(&globals_union, "generated/globals.h", "a");
+ strbuf_dump(&newtoys_h, "generated/newtoys.h", "w");
+}
+
+void generate_config_in() {
+ parse_help("Config.in");
+ parse_help("generated/Config.in");
+ strbuf_dump(&help_h, "generated/help.h", "w");
+}
+
+void parse_config() {
+ // Configu do not exists, create dummy .config
+ if (!access(".config", R_OK) == 0) {
+ f = xfopen(".config", "w");
+ fclose(f);
+ return;
+ }
+
+ // Parse .config
+ f = xfopen(".config", "r");
+ while (getline(&line, &linelen, f) > 0) {
+ if (strncmp(line, "CONFIG_", 7) == 0) {
+ // Config is ON
+ conf = line + 7;
+ pos = strchr(conf, '=');
+ if (pos) pos[0] = '\0';
+ strbuf_printf(&config_h, "#define CFG_%s 1\n", conf);
+ strbuf_printf(&config_h, "#define USE_%s(...) __VA_ARGS__\n", conf);
+ if (strncmp(conf, "TOYBOX_", 7) != 0) {
+ // Generate build_files
+ int config_len = strlen(conf);
+ for (i = 0; i < config_len; i++) {
+ conf[i] = tolower((unsigned char)conf[i]);
+ if (conf[i] == '\n' || conf[i] == ' ' || conf[i] == '=')
+ conf[i] = '\0';
+ }
+ snprintf(tmp, sizeof(tmp) - 1, "toys/%s.c", conf);
+ tmp[sizeof(tmp) - 1] = '\0';
+ if (access(tmp, R_OK) == 0)
+ strbuf_printf(&build_files, "%s\n", tmp);
+ }
+ continue;
+ }
+ if (strncmp(line, "# CONFIG_", 9) == 0) {
+ // Config is OFF
+ conf = line + 9;
+ pos = strchr(conf, ' ');
+ if (pos) pos[0] = '\0';
+ strbuf_printf(&config_h, "#define CFG_%s 0\n", conf);
+ strbuf_printf(&config_h, "#define USE_%s(...)\n", conf);
+ continue;
+ }
+ }
+ fclose(f);
+ strbuf_dump(&config_h, "generated/config.h", "w");
+ strbuf_dump(&build_files, "generated/build_files", "w");
+}
+
+
+int main(int argc, char **argv) {
+ if (argc < 2) {
+ fprintf(stderr, "Usage: %s toys/*.c\n", argv[0]);
+ exit(1);
+ }
+
+ generate_headers(argc, argv);
+ generate_config_in();
+ parse_config();
+
+ free(line);
+
+ return 0;
+}
diff --git a/scripts/genconfig.sh b/scripts/genconfig.sh
index f677954..c562e37 100755
--- a/scripts/genconfig.sh
+++ b/scripts/genconfig.sh
@@ -34,4 +34,12 @@ genconfig()
}
probeconfig > generated/Config.probed || rm generated/Config.probed
-genconfig > generated/Config.in || rm generated/Config.in
+
+if [ ! -x generated/genconf ]
+then
+ echo "Building generated/genconf"
+ $HOSTCC -ggdb -Wall -Wextra -Wno-sign-compare -I . -o generated/genconf scripts/genconf.c || exit 1
+fi
+
+echo "Extract configuration information from toys/*.c files..."
+generated/genconf toys/*.c || exit 1
diff --git a/scripts/make.sh b/scripts/make.sh
index f5bf0fd..e9afa68 100755
--- a/scripts/make.sh
+++ b/scripts/make.sh
@@ -4,95 +4,9 @@
source ./configure
-echo "Extract configuration information from toys/*.c files..."
scripts/genconfig.sh
-echo "Generate headers from toys/*.c..."
-
-# Create a list of all the applets toybox can provide. Note that the first
-# entry is out of order on purpose (the toybox multiplexer applet must be the
-# first element of the array). The rest must be sorted in alphabetical order
-# for fast binary search.
-
-function newtoys()
-{
- for i in toys/*.c
- do
- sed -n -e '1,/^config [A-Z]/s/^USE_/&/p' $i || exit 1
- done
-}
-echo "NEWTOY(toybox, NULL, 0)" > generated/newtoys.h
-newtoys | sed 's/\(.*TOY(\)\([^,]*\),\(.*\)/\2 \1\2,\3/' | sort -k 1,1 \
- | sed 's/[^ ]* //' >> generated/newtoys.h
-
-# Extract global structure definitions from toys/*.c
-
-function getglobals()
-{
- for i in toys/*.c
- do
- NAME="$(echo $i | sed 's at toys/\(.*\)\.c@\1@')"
-
- echo -e "// $i\n"
- sed -n -e '/^DEFINE_GLOBALS(/,/^)/b got;b;:got' \
- -e 's/^DEFINE_GLOBALS(/struct '"$NAME"'_data {/' \
- -e 's/^)/};/' -e 'p' $i
- done
-}
-
-GLOBSTRUCT="$(getglobals)"
-(
- echo "$GLOBSTRUCT"
- echo
- echo "extern union global_union {"
- echo "$GLOBSTRUCT" | sed -n 's/struct \(.*\)_data {/ struct \1_data \1;/p'
- echo "} this;"
-) > generated/globals.h
-
-# Only recreate generated/help.h if python is installed
-if [ ! -z "$(which python)" ] && [ ! -z "$(grep 'CONFIG_HELP=y' .config)" ]
-then
- echo "Extract help text from Config.in."
- scripts/config2help.py Config.in > generated/help.h || exit 1
-fi
-
-echo "Make generated/config.h from .config."
-
-# This long and roundabout sed invocation is to make old versions of sed happy.
-# New ones have '\n' so can replace one line with two without all the branches
-# and tedious mucking about with hold space.
-
-sed -n \
- -e 's/^# CONFIG_\(.*\) is not set.*/\1/' \
- -e 't notset' \
- -e 's/^CONFIG_\(.*\)=y.*/\1/' \
- -e 't isset' \
- -e 's/^CONFIG_\([^=]*\)=\(.*\)/#define CFG_\1 \2/p' \
- -e 'd' \
- -e ':notset' \
- -e 'h' \
- -e 's/.*/#define CFG_& 0/p' \
- -e 'g' \
- -e 's/.*/#define USE_&(...)/p' \
- -e 'd' \
- -e ':isset' \
- -e 'h' \
- -e 's/.*/#define CFG_& 1/p' \
- -e 'g' \
- -e 's/.*/#define USE_&(...) __VA_ARGS__/p' \
- .config > generated/config.h || exit 1
-
-# Extract a list of toys/*.c files to compile from the data in ".config" with
-# sed, sort, and tr:
-
-# 1) Grab the XXX part of all CONFIG_XXX entries, removing everything after the
-# second underline
-# 2) Sort the list, keeping only one of each entry.
-# 3) Convert to lower case.
-# 4) Remove toybox itself from the list (as that indicates global symbols).
-# 5) Add "toys/" prefix and ".c" suffix.
-
-TOYFILES=$(cat .config | sed -nre 's/^CONFIG_(.*)=y/\1/;t skip;b;:skip;s/_.*//;p' | sort -u | tr A-Z a-z | grep -v '^toybox$' | sed 's@\(.*\)@toys/\1.c@' )
+TOYFILES=$(cat generated/build_files 2>/dev/null)
echo "Compile toybox..."
--
1.7.5.1
More information about the Toybox
mailing list