From 14158b4127dba30466c50147b868a6a89702960b Mon Sep 17 00:00:00 2001 From: Bartosz Golaszewski Date: Tue, 17 Jun 2014 17:09:17 +0200 Subject: find: add optional support for '-exec ... {} +' function old new delta do_exec - 309 +309 parse_params 1416 1487 +71 find_main 342 406 +64 packed_usage 29958 30014 +56 func_exec 138 127 -11 ------------------------------------------------------------------------------ (add/remove: 1/0 grow/shrink: 3/1 up/down: 500/-11) Total: 489 bytes Signed-off-by: Bartosz Golaszewski Signed-off-by: Denys Vlasenko --- findutils/find.c | 156 +++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 141 insertions(+), 15 deletions(-) (limited to 'findutils') diff --git a/findutils/find.c b/findutils/find.c index 6d34f4d68..8ac3da7a0 100644 --- a/findutils/find.c +++ b/findutils/find.c @@ -137,6 +137,16 @@ //config: Support the 'find -exec' option for executing commands based upon //config: the files matched. //config: +//config:config FEATURE_FIND_EXEC_PLUS +//config: bool "Enable -exec ... {} +" +//config: default y +//config: depends on FEATURE_FIND_EXEC +//config: help +//config: Support the 'find -exec ... {} +' option for executing commands +//config: for all matched files at once. +//config: Without this option, -exec + is a synonym for -exec ; +//config: (IOW: it works correctly, but without expected speedup) +//config: //config:config FEATURE_FIND_USER //config: bool "Enable -user: username/uid matching" //config: default y @@ -319,6 +329,9 @@ //usage: "\n -exec CMD ARG ; Run CMD with all instances of {} replaced by" //usage: "\n file name. Fails if CMD exits with nonzero" //usage: ) +//usage: IF_FEATURE_FIND_EXEC_PLUS( +//usage: "\n -exec CMD ARG + Run CMD with {} replaced by list of file names" +//usage: ) //usage: IF_FEATURE_FIND_DELETE( //usage: "\n -delete Delete current file/directory. Turns on -depth option" //usage: ) @@ -337,8 +350,12 @@ # define FNM_CASEFOLD 0 #endif -#define dbg(...) ((void)0) -/* #define dbg(...) bb_error_msg(__VA_ARGS__) */ +#if 1 +# define dbg(...) ((void)0) +#else +# define dbg(...) bb_error_msg(__VA_ARGS__) +#endif + /* This is a NOEXEC applet. Be very careful! */ @@ -375,7 +392,20 @@ IF_FEATURE_FIND_CONTEXT(ACTS(context, security_context_t context;)) IF_FEATURE_FIND_PAREN( ACTS(paren, action ***subexpr;)) IF_FEATURE_FIND_PRUNE( ACTS(prune)) IF_FEATURE_FIND_DELETE( ACTS(delete)) -IF_FEATURE_FIND_EXEC( ACTS(exec, char **exec_argv; unsigned *subst_count; int exec_argc;)) +IF_FEATURE_FIND_EXEC( ACTS(exec, + char **exec_argv; /* -exec ARGS */ + unsigned *subst_count; + int exec_argc; /* count of ARGS */ + IF_FEATURE_FIND_EXEC_PLUS( + /* + * filelist is NULL if "exec ;" + * non-NULL if "exec +" + */ + char **filelist; + int filelist_idx; + int file_len; + ) + )) IF_FEATURE_FIND_GROUP( ACTS(group, gid_t gid;)) IF_FEATURE_FIND_LINKS( ACTS(links, char links_char; int links_count;)) @@ -452,7 +482,6 @@ static int exec_actions(action ***appp, const char *fileName, const struct stat return rc ^ TRUE; /* restore TRUE bit */ } - #if !FNM_CASEFOLD static char *strcpy_upcase(char *dst, const char *src) { @@ -576,17 +605,56 @@ ACTF(inum) } #endif #if ENABLE_FEATURE_FIND_EXEC -ACTF(exec) +static int do_exec(action_exec *ap, const char *fileName) { int i, rc; -#if ENABLE_USE_PORTABLE_CODE - char **argv = alloca(sizeof(char*) * (ap->exec_argc + 1)); -#else /* gcc 4.3.1 generates smaller code: */ - char *argv[ap->exec_argc + 1]; -#endif - for (i = 0; i < ap->exec_argc; i++) - argv[i] = xmalloc_substitute_string(ap->exec_argv[i], ap->subst_count[i], "{}", fileName); - argv[i] = NULL; /* terminate the list */ +# if ENABLE_FEATURE_FIND_EXEC_PLUS + int size = ap->exec_argc + ap->filelist_idx + 1; +# else + int size = ap->exec_argc + 1; +# endif +# if ENABLE_USE_PORTABLE_CODE + char **argv = alloca(sizeof(char*) * size); +# else /* gcc 4.3.1 generates smaller code: */ + char *argv[size]; +# endif + char **pp = argv; + + for (i = 0; i < ap->exec_argc; i++) { + const char *arg = ap->exec_argv[i]; + +# if ENABLE_FEATURE_FIND_EXEC_PLUS + if (ap->filelist) { + /* Handling "-exec +" + * Only one exec_argv[i] has substitution in it. + * Expand that one exec_argv[i] into file list. + */ + if (ap->subst_count[i] == 0) { + *pp++ = xstrdup(arg); + } else { + int j = 0; + while (ap->filelist[j]) { + *pp++ = xmalloc_substitute_string(arg, 1, "{}", ap->filelist[j]); + free(ap->filelist[j]); + j++; + } + } + } else +# endif + { + /* Handling "-exec ;" */ + *pp++ = xmalloc_substitute_string(arg, ap->subst_count[i], "{}", fileName); + } + } + *pp = NULL; /* terminate the list */ + +# if ENABLE_FEATURE_FIND_EXEC_PLUS + if (ap->filelist) { + ap->filelist[0] = NULL; + ap->filelist_idx = 0; + ap->file_len = 0; + } +# endif rc = spawn_and_wait(argv); if (rc < 0) @@ -597,6 +665,48 @@ ACTF(exec) free(argv[i++]); return rc == 0; /* return 1 if exitcode 0 */ } +ACTF(exec) +{ +# if ENABLE_FEATURE_FIND_EXEC_PLUS + if (ap->filelist) { + int rc = 0; + + /* If we have lots of files already, exec the command */ + if (ap->file_len >= 32*1024) + rc = do_exec(ap, NULL); + + ap->file_len += strlen(fileName) + sizeof(char*) + 1; + ap->filelist = xrealloc_vector(ap->filelist, 8, ap->filelist_idx); + ap->filelist[ap->filelist_idx++] = xstrdup(fileName); + return rc == 0; /* return 1 if exitcode 0 */ + } +# endif + return do_exec(ap, fileName); +} +# if ENABLE_FEATURE_FIND_EXEC_PLUS +static int flush_exec_plus(void) +{ + action *ap; + action **app; + action ***appp = G.actions; + while ((app = *appp++) != NULL) { + while ((ap = *app++) != NULL) { + if (ap->f == (action_fp)func_exec) { + action_exec *ae = (void*)ap; + if (ae->filelist_idx != 0) { + int rc = do_exec(ae, NULL); +# if ENABLE_FEATURE_FIND_NOT + if (ap->invert) rc = !rc; +# endif + if (rc) + return rc; + } + } + } + } + return 0; +} +# endif #endif #if ENABLE_FEATURE_FIND_USER ACTF(user) @@ -1037,6 +1147,7 @@ static action*** parse_params(char **argv) else if (parm == PARM_exec) { int i; action_exec *ap; + IF_FEATURE_FIND_EXEC_PLUS(int all_subst = 0;) dbg("%d", __LINE__); G.need_print = 0; ap = ALLOC_ACTION(exec); @@ -1049,10 +1160,13 @@ static action*** parse_params(char **argv) // executes "echo Foo >FILENAME<", // find -exec echo Foo ">{}<" "+" // executes "echo Foo FILENAME1 FILENAME2 FILENAME3...". - // TODO (so far we treat "+" just like ";") if ((argv[0][0] == ';' || argv[0][0] == '+') && argv[0][1] == '\0' ) { +# if ENABLE_FEATURE_FIND_EXEC_PLUS + if (argv[0][0] == '+') + ap->filelist = xzalloc(sizeof(ap->filelist[0])); +# endif break; } argv++; @@ -1062,8 +1176,17 @@ static action*** parse_params(char **argv) bb_error_msg_and_die(bb_msg_requires_arg, arg); ap->subst_count = xmalloc(ap->exec_argc * sizeof(int)); i = ap->exec_argc; - while (i--) + while (i--) { ap->subst_count[i] = count_strstr(ap->exec_argv[i], "{}"); + IF_FEATURE_FIND_EXEC_PLUS(all_subst += ap->subst_count[i];) + } +# if ENABLE_FEATURE_FIND_EXEC_PLUS + /* + * coreutils expects {} to appear only once in "-exec +" + */ + if (all_subst != 1 && ap->filelist) + bb_error_msg_and_die("only one '{}' allowed for -exec +"); +# endif } #endif #if ENABLE_FEATURE_FIND_PAREN @@ -1335,8 +1458,11 @@ int find_main(int argc UNUSED_PARAM, char **argv) 0) /* depth */ ) { status = EXIT_FAILURE; + goto out; } } + IF_FEATURE_FIND_EXEC_PLUS(status = flush_exec_plus();) +out: return status; } -- cgit v1.2.3-55-g6feb From 6be3a5242ce4855734a4cdd5770b6ea7adaf2b3d Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Thu, 19 Jun 2014 11:32:11 +0200 Subject: find: exit code fixes for find -exec function old new delta func_exec 127 100 -27 Signed-off-by: Denys Vlasenko --- findutils/find.c | 17 +++++++++-------- testsuite/find.tests | 26 ++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 8 deletions(-) (limited to 'findutils') diff --git a/findutils/find.c b/findutils/find.c index 8ac3da7a0..493f72e61 100644 --- a/findutils/find.c +++ b/findutils/find.c @@ -634,6 +634,7 @@ static int do_exec(action_exec *ap, const char *fileName) } else { int j = 0; while (ap->filelist[j]) { + /* 2nd arg here should be ap->subst_count[i], but it is always 1: */ *pp++ = xmalloc_substitute_string(arg, 1, "{}", ap->filelist[j]); free(ap->filelist[j]); j++; @@ -669,16 +670,16 @@ ACTF(exec) { # if ENABLE_FEATURE_FIND_EXEC_PLUS if (ap->filelist) { - int rc = 0; + int rc; + ap->filelist = xrealloc_vector(ap->filelist, 8, ap->filelist_idx); + ap->filelist[ap->filelist_idx++] = xstrdup(fileName); + ap->file_len += strlen(fileName) + sizeof(char*) + 1; /* If we have lots of files already, exec the command */ + rc = 1; if (ap->file_len >= 32*1024) rc = do_exec(ap, NULL); - - ap->file_len += strlen(fileName) + sizeof(char*) + 1; - ap->filelist = xrealloc_vector(ap->filelist, 8, ap->filelist_idx); - ap->filelist[ap->filelist_idx++] = xstrdup(fileName); - return rc == 0; /* return 1 if exitcode 0 */ + return rc; } # endif return do_exec(ap, fileName); @@ -698,8 +699,8 @@ static int flush_exec_plus(void) # if ENABLE_FEATURE_FIND_NOT if (ap->invert) rc = !rc; # endif - if (rc) - return rc; + if (rc == 0) + return 1; } } } diff --git a/testsuite/find.tests b/testsuite/find.tests index 345d1e82e..f041106c3 100755 --- a/testsuite/find.tests +++ b/testsuite/find.tests @@ -15,6 +15,32 @@ testing "find -type f" \ "./testfile\n" \ "" "" +optional FEATURE_FIND_EXEC +testing "find -exec exitcode 1" \ + "cd find.tempdir && find testfile -exec true {} \; 2>&1; echo \$?" \ + "0\n" \ + "" "" +SKIP= +optional FEATURE_FIND_EXEC_PLUS +testing "find -exec exitcode 2" \ + "cd find.tempdir && find testfile -exec true {} + 2>&1; echo \$?" \ + "0\n" \ + "" "" +SKIP= +# Surprisingly, "-exec false ;" results in exitcode 0! "-exec false +" is different!!! +optional FEATURE_FIND_EXEC +testing "find -exec exitcode 3" \ + "cd find.tempdir && find testfile -exec false {} \; 2>&1; echo \$?" \ + "0\n" \ + "" "" +SKIP= +optional FEATURE_FIND_EXEC_PLUS +testing "find -exec exitcode 4" \ + "cd find.tempdir && find testfile -exec false {} + 2>&1; echo \$?" \ + "1\n" \ + "" "" +SKIP= + # testing "description" "command" "result" "infile" "stdin" rm -rf find.tempdir -- cgit v1.2.3-55-g6feb From f92f1d0181853b989f9377debb56902e3e21c9a8 Mon Sep 17 00:00:00 2001 From: Denys Vlasenko Date: Sun, 22 Jun 2014 13:54:40 +0200 Subject: find: use sysconf(_SC_ARG_MAX) to determine the command-line size limit The find utility uses a hardcoded value of 32 * 1024 as the limit of the command-line length when calling 'find -exec ... {} +'. This results in over 4 times more execve() calls than in coreutils' find. This patch uses the limit defined in system headers. Based on the patch by Bartosz Golaszewski . Signed-off-by: Denys Vlasenko --- findutils/find.c | 4 +++- findutils/xargs.c | 26 ++++++++++---------------- include/libbb.h | 8 ++++++++ libbb/Kbuild.src | 1 + libbb/sysconf.c | 16 ++++++++++++++++ 5 files changed, 38 insertions(+), 17 deletions(-) create mode 100644 libbb/sysconf.c (limited to 'findutils') diff --git a/findutils/find.c b/findutils/find.c index 493f72e61..56a7ed3ab 100644 --- a/findutils/find.c +++ b/findutils/find.c @@ -419,6 +419,7 @@ struct globals { smallint need_print; smallint xdev_on; recurse_flags_t recurse_flags; + IF_FEATURE_FIND_EXEC_PLUS(unsigned max_argv_len;) } FIX_ALIASING; #define G (*(struct globals*)&bb_common_bufsiz1) #define INIT_G() do { \ @@ -428,6 +429,7 @@ struct globals { /* we have to zero it out because of NOEXEC */ \ memset(&G, 0, sizeof(G)); \ IF_FEATURE_FIND_MAXDEPTH(G.minmaxdepth[1] = INT_MAX;) \ + IF_FEATURE_FIND_EXEC_PLUS(G.max_argv_len = bb_arg_max() - 2048;) \ G.need_print = 1; \ G.recurse_flags = ACTION_RECURSE; \ } while (0) @@ -677,7 +679,7 @@ ACTF(exec) ap->file_len += strlen(fileName) + sizeof(char*) + 1; /* If we have lots of files already, exec the command */ rc = 1; - if (ap->file_len >= 32*1024) + if (ap->file_len >= G.max_argv_len) rc = do_exec(ap, NULL); return rc; } diff --git a/findutils/xargs.c b/findutils/xargs.c index 0ba5b566d..76c4747fe 100644 --- a/findutils/xargs.c +++ b/findutils/xargs.c @@ -523,12 +523,7 @@ int xargs_main(int argc, char **argv) argc++; } - /* -s NUM default. fileutils-4.4.2 uses 128k, but I heasitate - * to use such a big value - first need to change code to use - * growable buffer instead of fixed one. - */ - n_max_chars = 32 * 1024; - /* Make smaller if system does not allow our default value. + /* * The Open Group Base Specifications Issue 6: * "The xargs utility shall limit the command line length such that * when the command line is invoked, the combined argument @@ -536,16 +531,15 @@ int xargs_main(int argc, char **argv) * in the System Interfaces volume of IEEE Std 1003.1-2001) * shall not exceed {ARG_MAX}-2048 bytes". */ - { - long arg_max = 0; -#if defined _SC_ARG_MAX - arg_max = sysconf(_SC_ARG_MAX) - 2048; -#elif defined ARG_MAX - arg_max = ARG_MAX - 2048; -#endif - if (arg_max > 0 && n_max_chars > arg_max) - n_max_chars = arg_max; - } + n_max_chars = bb_arg_max(); + if (n_max_chars > 32 * 1024) + n_max_chars = 32 * 1024; + /* + * POSIX suggests substracting 2048 bytes from sysconf(_SC_ARG_MAX) + * so that the process may safely modify its environment. + */ + n_max_chars -= 2048; + if (opt & OPT_UPTO_SIZE) { n_max_chars = xatou_range(max_chars, 1, INT_MAX); } diff --git a/include/libbb.h b/include/libbb.h index a1a0dc18c..fa69a7fe6 100644 --- a/include/libbb.h +++ b/include/libbb.h @@ -731,6 +731,14 @@ extern void *xmalloc_open_read_close(const char *filename, size_t *maxsz_p) FAST /* Never returns NULL */ extern void *xmalloc_xopen_read_close(const char *filename, size_t *maxsz_p) FAST_FUNC RETURNS_MALLOC; +#if defined ARG_MAX +# define bb_arg_max() ((unsigned)ARG_MAX) +#elif defined _SC_ARG_MAX +unsigned bb_arg_max(void) FAST_FUNC; +#else +# define bb_arg_max() ((unsigned)(32 * 1024)) +#endif + #define SEAMLESS_COMPRESSION (0 \ || ENABLE_FEATURE_SEAMLESS_XZ \ || ENABLE_FEATURE_SEAMLESS_LZMA \ diff --git a/libbb/Kbuild.src b/libbb/Kbuild.src index 6578d1171..62680bd52 100644 --- a/libbb/Kbuild.src +++ b/libbb/Kbuild.src @@ -92,6 +92,7 @@ lib-y += skip_whitespace.o lib-y += speed_table.o lib-y += str_tolower.o lib-y += strrstr.o +lib-y += sysconf.o lib-y += time.o lib-y += trim.o lib-y += u_signal_names.o diff --git a/libbb/sysconf.c b/libbb/sysconf.c new file mode 100644 index 000000000..c5fa5e001 --- /dev/null +++ b/libbb/sysconf.c @@ -0,0 +1,16 @@ +/* vi: set sw=4 ts=4: */ +/* + * Various system configuration helpers. + * + * Copyright (C) 2014 Bartosz Golaszewski + * + * Licensed under GPLv2 or later, see file LICENSE in this source tree. + */ +#include "libbb.h" + +#if defined _SC_ARG_MAX +unsigned FAST_FUNC bb_arg_max(void) +{ + return sysconf(_SC_ARG_MAX); +} +#endif -- cgit v1.2.3-55-g6feb