[Toybox] [PATCH] Generate all build files with single program.
Rob Landley
rob at landley.net
Fri Mar 16 19:57:45 PDT 2012
On 03/16/2012 05:05 PM, Georgi Chorbadzhiyski wrote:
> This patch removes all external tools from the build system. Now
> everything in generated/ directory is build with single program.
> The effect is that the build no longer requires sed, tr, sort and
> python (for rebuilding the help texts). There is no need to ship
> generated/help.h since it can be rebuild on the host without python.
Cool!
Generating help.h in C (and eliminating the need for python) is
something I've been meaning to do for a while. But building the list of
C files to compile... in C... seems unnecessarily brittle somehow.
> There is also a nice speedup. On my machine building allyesconfig
> is ~1.8 seconds faster, it goes down from 11.3s to 0m9.5s.
> ---
> Makefile | 7 +-
> scripts/config2help.py | 54 ---------
> scripts/config2help.sh | 54 ---------
> scripts/genconf.c | 309 ++++++++++++++++++++++++++++++++++++++++++++++++
> scripts/genconfig.sh | 10 ++-
> scripts/make.sh | 88 +--------------
> 6 files changed, 324 insertions(+), 198 deletions(-)
> delete mode 100755 scripts/config2help.py
> delete mode 100755 scripts/config2help.sh
> create mode 100644 scripts/genconf.c
>
> diff --git a/Makefile b/Makefile
> index 7079f82..4aec0ff 100644
> --- a/Makefile
> +++ b/Makefile
> @@ -17,6 +17,9 @@ generated/Config.in: toys/*.c scripts/genconfig.sh
>
> HOSTCC?=cc
>
> +.config:
> + $(MAKE) defconfig
> +
> # Development targets
> baseline: toybox_unstripped
> @cp toybox_unstripped toybox_old
> @@ -42,10 +45,10 @@ uninstall:
> clean::
> rm -rf toybox toybox_unstripped generated/config.h generated/Config.in \
> generated/newtoys.h generated/globals.h instlist testdir \
> - generated/Config.probed
> + generated/Config.probed generated/help.h generated/build_files
>
> distclean: clean
> - rm -f toybox_old .config* generated/help.h
> + rm -f toybox_old .config* generated/genconf
>
> test: tests
>
> diff --git a/scripts/config2help.py b/scripts/config2help.py
> deleted file mode 100755
> index 2573d08..0000000
> --- a/scripts/config2help.py
> +++ /dev/null
> @@ -1,54 +0,0 @@
> -#!/usr/bin/python
> -
> -import os,sys
> -
> -def zapquotes(str):
> - if str[0]=='"': str = str[1:str.rfind('"')]
> - return str
> -
> -def escapequotes(str):
> - return str.strip().replace("\\","\\\\").replace('"','\\"')
> -
> -helplen = morelines = 0
> -out = sys.stdout
> -
> -def readfile(filename):
> - global helplen, morelines
> - #sys.stderr.write("Reading %s\n" % filename)
> - try:
> - lines = open(filename).read().split("\n")
> - except IOError:
> - sys.stderr.write("File %s missing\n" % filename)
> - return
> - config = None
> - description = None
> - for i in lines:
> - if helplen:
> - i = i.expandtabs()
> - if not len(i) or i[:helplen].isspace():
> - if morelines: out.write('\\n')
> - morelines = 1
> - out.write(escapequotes(i))
> - continue
> - else:
> - helplen = morelines = 0
> - out.write('"\n')
> -
> - words = i.strip().split(None,1)
> - if not len(words): continue
> -
> - if words[0] in ("config", "menuconfig"):
> - config = words[1]
> - description = ""
> - elif words[0] in ("bool", "boolean", "tristate", "string", "hex", "int"):
> - if len(words)>1: description = zapquotes(words[1])
> - elif words[0]=="prompt":
> - description = htmlescape(zapquotes(words[1]))
> - elif words[0] in ("help", "---help---"):
> - out.write('#define help_%s "' % config.lower())
> - helplen = len(i[:i.find(words[0])].expandtabs())
> - elif words[0] == "source": readfile(zapquotes(words[1]))
> - elif words[0] in ("default","depends", "select", "if", "endif", "#", "comment", "menu", "endmenu"): pass
> -
> -readfile(sys.argv[1])
> -if helplen: out.write('"\n')
> diff --git a/scripts/config2help.sh b/scripts/config2help.sh
> deleted file mode 100755
> index 63d6c6f..0000000
> --- a/scripts/config2help.sh
> +++ /dev/null
> @@ -1,54 +0,0 @@
> -#!/bin/bash -e
> -
> -function firstmajor() {
> - declare -i n j=$1
> - test $j -gt 0 || return -1
> - for j in $@; do
> - if [ $j -gt $1 ]; then
> - echo $j
> - return 0
> - fi
> - done
> - return 1
> -}; export -f firstmajor
> -
> -function print_h() {
> - declare -i i c=$2 s=$3 e=$4
> - local str="$(echo "$1" | head -n$c | tail -n1 | sed -e "s,config[\t ]*,,")"
> - echo -n "#define help_"$str" \"" | tr [A-Z] [a-z]
> - echo -n "$1\\n" | head -n$e | tail -n$[e-s+1] | sed -e "s,\$,\r," | tr \\n\\r n\\
> - echo \"
> -}; export -f print_h
> -
> -file="$1"
> -if test "$0" != "bash"; then
> - if test -r "$file"; then
> -# echo "$file..." >&2
> - filetxt="$(sed -e "s,^[\t ]*,," -e "s,\([\"\\\\]\),\\\\\\1,g" "$file")"
> - helps=$(echo "$filetxt" | egrep -ne "^help *" -e "^---help--- *" | cut -d\: -f1)
> - configs=$(echo "$filetxt" | egrep -ne "^config *" | cut -d\: -f1)
> - endmenus=$(echo "$filetxt" | egrep -ne "^endmenu *" | cut -d\: -f1)
> - let last=$(echo "$filetxt" | wc -l)+2
> -
> - declare -i i c s e
> - for i in $configs; do
> -# echo -n "c:$i" >&2
> - s=$(firstmajor $i $helps)
> - test $s -gt 0
> - e=$(firstmajor $s $configs || firstmajor $s $endmenus $last)
> - let s++ e-=2
> - test $e -ge $s
> -# echo " s:$s e:$e" >&2
> - print_h "$filetxt" $i $s $e
> - done
> - for fle in $(cat "$file" | egrep -e "^[ \t]*source " | sed -e "s,^[ \t]*source *\(.*\),\\1,"); do
> - $0 $fle
> - done
> - else
> - echo
> - echo "USAGE EXAMPLE: $(basename $0) Config.in > generated/help.h"
> - echo
> - false
> - fi | sed -e "s,\\\\n\\\\n\"$,\\\\n\","
> -fi
> -
> diff --git a/scripts/genconf.c b/scripts/genconf.c
> new file mode 100644
> index 0000000..8346dee
> --- /dev/null
> +++ b/scripts/genconf.c
> @@ -0,0 +1,309 @@
> +/* vi: set ts=4 :*/
> +/*
> + * Generate toybox generated/{Config.in,config.h,globals.h,newtoys.h,help.h}
> + *
> + * Copyright 2012 Georgi Chorbadzhiyski <georgi at unixsol.org>
> + */
> +__attribute__ ((format(printf, 2, 3)))
> +void strbuf_printf(struct strbuf *s, const char *fmt, ...) {
This is a purely local function, the __attribute__ tag is overkill.
> + char line[4096];
> +
> + va_list args;
> + va_start(args, fmt);
> + vsnprintf(line, sizeof(line) - 1, fmt, args);
> + va_end(args);
> +
> + line[sizeof(line) - 1] = '\0';
> + strbuf_add(s, line);
> +}
> +
> +void strbuf_add_escaped(struct strbuf *s, char *text) {
> + int len = strlen(text), i;
> + if (len == 0)
> + strbuf_add(s, "\\n");
> + for (i = 0; i < len; i++) {
> + char c[2] = { text[i], 0 };
> + switch (text[i]) {
> + case '\n': strbuf_add(s, "\\n"); break;
> + case '\t': strbuf_add(s, " "); break;
> + case '\\': strbuf_add(s, "\\\\"); break;
> + case '"' : strbuf_add(s, "\\\""); break;
> + default : strbuf_add(s, c); break;
> + }
> + }
> +}
> +
> +void strbuf_dump(struct strbuf *s, char *filename, char *mode) {
> + FILE *fh;
> + if (!s->data) {
> + unlink(filename);
> + return;
> + }
> + fh = xfopen(filename, mode);
> + fwrite(s->data, s->len, 1, fh);
> + fclose(fh);
> + free(s->data);
> +}
> +
> +void parse_help(char *filename) {
> + int i, nitems = 0;
> + char *config_name = NULL;
> +
> + f = xfopen(filename, "r");
> + while (getline(&line, &linelen, f) > 0) {
> + if (line[0] == '#')
> + continue;
> + if (strncmp(line, "config ", 7) == 0) {
> + in_config = 1;
> + in_help = 0;
> + free(config_name);
> + config_name = strdup(line + 7);
> + int config_len = strlen(config_name);
> + for (i = 0; i < config_len; i++) {
> + config_name[i] = tolower((unsigned char)config_name[i]);
> + if (config_name[i] == '\n')
> + config_name[i] = '\0';
> + }
> + // Close already started help line
> + if (nitems)
> + strbuf_add(&help_h, "\"\n");
> + continue;
> + }
> + if (strncmp(line, "endmenu", 7) == 0) {
> + in_config = 0;
> + in_help = 0;
> + }
> + if (!in_config)
> + continue;
> +
> + if (!in_help) {
> + if (strncmp(line, "\thelp", 5) == 0 || strncmp(line, " help", 8) == 0) {
> + in_help = 1;
> + nitems++;
> + strbuf_printf(&help_h, "#define help_%s \"", config_name);
> + }
> + continue;
> + } else {
> + char *tmpline = line;
> + while (isspace(*tmpline))
> + tmpline++;
> + strbuf_add_escaped(&help_h, tmpline);
> + continue;
> + }
> + }
> + strbuf_add(&help_h, "\"\n");
I was trying to detect sub-options, and also merge the "usage:" lines.
(This turn out to be a bit fiddly, you can use the "depends" lines which
can have "depends on ONE && TWO" so there's some parsing to do, or you
can depend on CONFIG_WALRUS and CONFIG_WALRUS_SUBOPT starting with the
same prefix, which gets fiddly when CONFIG_WALRUS has multiple options
and how do you tell that CONFIG_PIVOT_ROOT isn't a subopt for CONFIG_PIVOT?)
> +void generate_headers(int argc, char **argv) {
> + // Parse toys
> + strbuf_add(&newtoys_h, "NEWTOY(toybox, NULL, 0)\n");
> + strbuf_add(&globals_union, "extern union global_union {\n");
It matters that these be in alphabetical order (for the binary search),
and we can't depend on the filesystem doing it for us. Where are you
sorting this?
I'll have to look at this more closely in the morning...
Rob
1331953065.0
More information about the Toybox
mailing list