First implementation for the accumulator capture

8 files changed, 131 insertions, 22 deletions

diff --git a/HISTORY b/HISTORY
index 7fa72e6..a250709 100644
--- a/HISTORY
+++ b/HISTORY
@@ -2,9 +2,12 @@ HISTORY for LPeg 1.1.0
 
 * Changes from version 1.0.2 to 1.1.0
   ---------------------------------
+  + accumulator capture
   + UTF-8 ranges
   + Larger limit for number of rules in a grammar
+  + Larger limit for number of captures in a match
   + bug fixes
+  + other small improvements
 
 * Changes from version 1.0.1 to 1.0.2
   ---------------------------------
diff --git a/lpcap.c b/lpcap.c
index ec17d23..4750f01 100644
--- a/lpcap.c
+++ b/lpcap.c
@@ -231,6 +231,22 @@ static int functioncap (CapState *cs) {
 
 
 /*
+** Accumulator capture
+*/
+static int accumulatorcap (CapState *cs) {
+  lua_State *L = cs->L;
+  int n;
+  if (lua_gettop(L) < cs->firstcap)
+    luaL_error(L, "no previous value for accumulator capture");
+  pushluaval(cs);  /* push function */
+  lua_insert(L, -2);  /* previous value becomes first argument */
+  n = pushnestedvalues(cs, 0);  /* push nested captures */
+  lua_call(L, n + 1, 1);  /* call function */
+  return 0;  /* did not add any extra value */
+}
+
+
+/*
 ** Select capture
 */
 static int numcap (CapState *cs) {
@@ -422,13 +438,16 @@ static int addonestring (luaL_Buffer *b, CapState *cs, const char *what) {
     case Csubst:
       substcap(b, cs);  /* add capture directly to buffer */
       return 1;
+    case Cacc:  /* accumulator capture? */
+      return luaL_error(cs->L, "accumulator capture inside substitution capture");
     default: {
       lua_State *L = cs->L;
       int n = pushcapture(cs);
       if (n > 0) {
         if (n > 1) lua_pop(L, n - 1);  /* only one result */
         if (!lua_isstring(L, -1))
-          luaL_error(L, "invalid %s value (a %s)", what, luaL_typename(L, -1));
+          return luaL_error(L, "invalid %s value (a %s)",
+                               what, luaL_typename(L, -1));
         luaL_addvalue(b);
       }
       return n;
@@ -512,6 +531,7 @@ static int pushcapture (CapState *cs) {
     case Cbackref: res = backrefcap(cs); break;
     case Ctable: res = tablecap(cs); break;
     case Cfunction: res = functioncap(cs); break;
+    case Cacc: res = accumulatorcap(cs); break;
     case Cnum: res = numcap(cs); break;
     case Cquery: res = querycap(cs); break;
     case Cfold: res = foldcap(cs); break;
@@ -537,9 +557,11 @@ int getcaptures (lua_State *L, const char *s, const char *r, int ptop) {
     CapState cs;
     cs.ocap = cs.cap = capture; cs.L = L; cs.reclevel = 0;
     cs.s = s; cs.valuecached = 0; cs.ptop = ptop;
+    cs.firstcap = lua_gettop(L) + 1;  /* where first value (if any) will go */
     do {  /* collect their values */
       n += pushcapture(&cs);
     } while (!isclosecap(cs.cap));
+    assert(lua_gettop(L) - cs.firstcap == n - 1);
   }
   if (n == 0) {  /* no capture values? */
     lua_pushinteger(L, r - s + 1);  /* return only end position */
diff --git a/lpcap.h b/lpcap.h
index 10539a0..e72d913 100644
--- a/lpcap.h
+++ b/lpcap.h
@@ -16,6 +16,7 @@ typedef enum CapKind {
   Csimple,  /* next node is pattern */
   Ctable,  /* next node is pattern */
   Cfunction,  /* ktable[key] is function; next node is pattern */
+  Cacc,  /* ktable[key] is function; next node is pattern */
   Cquery,  /* ktable[key] is table; next node is pattern */
   Cstring,  /* ktable[key] is string; next node is pattern */
   Cnum,  /* numbered capture; 'key' is number of value to return */
@@ -38,7 +39,8 @@ typedef struct CapState {
   Capture *cap;  /* current capture */
   Capture *ocap;  /* (original) capture list */
   lua_State *L;
-  int ptop;  /* index of last argument to 'match' */
+  int ptop;  /* stack index of last argument to 'match' */
+  int firstcap;  /* stack index of first capture pushed in the stack */
   const char *s;  /* original string */
   int valuecached;  /* value stored in cache slot */
   int reclevel;  /* recursion level */
diff --git a/lpeg.html b/lpeg.html
index f50d327..9faa1c7 100644
--- a/lpeg.html
+++ b/lpeg.html
@@ -638,6 +638,10 @@ or no value when <code>number</code> is zero.</td></tr>
 <tr><td><a href="#cap-func"><code>patt / function</code></a></td>
   <td>the returns of <code>function</code> applied to the captures
       of <code>patt</code></td></tr>
+<tr><td><a href="#cap-rep"><code>patt % function</code></a></td>
+  <td>the return of <code>function</code> applied to the previous
+      capture plus the captures of <code>patt</code></td></tr>;
+      the returned value becomes the value of the previous capture
 <tr><td><a href="#matchtime"><code>lpeg.Cmt(patt, function)</code></a></td>
   <td>the returns of <code>function</code> applied to the captures
       of <code>patt</code>; the application is done at match time</td></tr>
@@ -889,6 +893,75 @@ there is no captured value.
 </p>
 
 
+<h3><a name="cap-rep"></a><code>patt % function</code></h3>
+<p>
+Creates an <em>accumulator capture</em>.
+This pattern behaves similarly to a
+<a href="cap-func">function capture</a>,
+with the following differences:
+The last captured value is added as a first argument to
+the call;
+the return of the function is adjusted to one single value;
+that value becomes the last captured value.
+</p>
+
+<p>
+As an example,
+consider the following code fragment:
+</p>
+<pre class="example">
+local name = lpeg.C(lpeg.R("az")^1)
+local p = name * (lpeg.P("^") % string.upper)^-1
+print(p:match("count"))    --&gt; count
+print(p:match("count^"))   --&gt; COUNT
+</pre>
+<p>
+In the first match,
+the accumulator capture does not match,
+and so the match results in its first capture, a name.
+In the second match,
+the accumulator capture matches,
+so the function <code>string.upper</code>
+is called with the previous capture (created by <code>name</code>)
+plus the string <code>"^"</code>;
+the function ignores its second argument and returns the first argument
+changed to upper case;
+that value then becomes the first and only
+capture value created by the match.
+</p>
+-- matches a numeral and captures its numerical value
+number = lpeg.R"09"^1 / tonumber
+
+-- auxiliary function to add two numbers
+function add (acc, newvalue) return acc + newvalue end
+
+-- matches a list of numbers, adding their values
+sum = number * ("," * number % add)^0
+
+-- example of use
+print(sum:match("10,30,43"))   --&gt; 83
+</pre>
+<p>
+First, the initial <code>number</code> captures a number;
+that first capture will play the role of an accumulator.
+Then, each time <code>number</code> matches inside the loop
+there is a accumulator capture:
+It calls <code>add</code> with the current value of the accumulator
+and the value of the new number,
+and their sum replaces the value of the accumulator.
+At the end of the match,
+the accumulator with all sums is the final value.
+</p>
+
+<p>
+Due to the nature of this capture,
+you should avoid using it in places where it is not clear
+what is its "previous" capture.
+Due to implementation details,
+you should not use this capture inside a substitution capture.
+</p>
+
+
 <h3><a name="matchtime"></a><code>lpeg.Cmt(patt, function)</code></h3>
 <p>
 Creates a <em>match-time capture</em>.
@@ -968,19 +1041,17 @@ lpeg.locale(lpeg)   -- adds locale entries into 'lpeg' table
 local space = lpeg.space^0
 local name = lpeg.C(lpeg.alpha^1) * space
 local sep = lpeg.S(",;") * space
-local pair = lpeg.Cg(name * "=" * space * name) * sep^-1
-local list = lpeg.Cf(lpeg.Ct("") * pair^0, rawset)
+local pair = name * "=" * space * name * sep^-1
+local list = lpeg.Ct("") * (pair % rawset)^0
 t = list:match("a=b, c = hi; next = pi")  --> { a = "b", c = "hi", next = "pi" }
 </pre>
 <p>
 Each pair has the format <code>name = name</code> followed by
 an optional separator (a comma or a semicolon).
-The <code>pair</code> pattern encloses the pair in a group pattern,
-so that the names become the values of a single capture.
-The <code>list</code> pattern then folds these captures.
+The <code>list</code> pattern then <em>folds</em> these captures.
 It starts with an empty table,
 created by a table capture matching an empty string;
-then for each capture (a pair of names) it applies <code>rawset</code>
+then for each a pair of names it applies <code>rawset</code>
 over the accumulator (the table) and the capture values (the pair of names).
 <code>rawset</code> returns the table itself,
 so the accumulator is always the table.
@@ -1295,8 +1366,8 @@ end
 -- Grammar
 local V = lpeg.V
 G = lpeg.P{ "Exp",
-  Exp = lpeg.Cf(V"Term" * lpeg.Cg(TermOp * V"Term")^0, eval);
-  Term = lpeg.Cf(V"Factor" * lpeg.Cg(FactorOp * V"Factor")^0, eval);
+  Exp = V"Term" * (TermOp * V"Term" % eval)^0;
+  Term = V"Factor" * (FactorOp * V"Factor" % eval)^0;
   Factor = Number / tonumber + Open * V"Exp" * Close;
 }
 
@@ -1304,7 +1375,7 @@ G = lpeg.P{ "Exp",
 print(lpeg.match(G, "3 + 5*9 / (1+1) - 12"))   --> 13.5
 </pre>
 <p>
-Note the use of the fold (accumulator) capture.
+Note the use of the accumulator capture.
 To compute the value of an expression,
 the accumulator starts with the value of the first term,
 and then applies <code>eval</code> over
diff --git a/lpprint.c b/lpprint.c
index 54a3da7..1142376 100644
--- a/lpprint.c
+++ b/lpprint.c
@@ -60,7 +60,7 @@ static void printTcharset (TTree *tree) {
 static const char *capkind (int kind) {
   const char *const modes[] = {
     "close", "position", "constant", "backref",
-    "argument", "simple", "table", "function",
+    "argument", "simple", "table", "function", "replace",
     "query", "string", "num", "substitution", "fold",
     "runtime", "group"};
   return modes[kind];
@@ -147,7 +147,6 @@ void printpatt (Instruction *p) {
 }
 
 
-#if defined(LPEG_DEBUG)
 static void printcap (Capture *cap) {
   printf("%s (idx: %d - size: %d) -> %p\n",
          capkind(cap->kind), cap->idx, cap->siz, cap->s);
@@ -160,7 +159,6 @@ void printcaplist (Capture *cap, Capture *limit) {
     printcap(cap);
   printf("=======\n");
 }
-#endif
 
 /* }====================================================== */
 
diff --git a/lptree.c b/lptree.c
index 0b2d824..330471a 100644
--- a/lptree.c
+++ b/lptree.c
@@ -19,7 +19,7 @@
 const byte numsiblings[] = {
   0, 0, 0,      /* char, set, any */
   0, 0, 0,      /* true, false, utf-range */
-  1,            /* rep */
+  1,            /* acc */
   2, 2,         /* seq, choice */
   1, 1,         /* not, and */
   0, 0, 2, 1, 1,  /* call, opencall, rule, prerule, grammar */
@@ -850,6 +850,11 @@ static int lp_divcapture (lua_State *L) {
 }
 
 
+static int lp_acccapture (lua_State *L) {
+  return capture_aux(L, Cacc, 2);
+}
+
+
 static int lp_substcapture (lua_State *L) {
   return capture_aux(L, Csubst, 0);
 }
@@ -1250,7 +1255,7 @@ static int lp_match (lua_State *L) {
   int ptop = lua_gettop(L);
   lua_pushnil(L);  /* initialize subscache */
   lua_pushlightuserdata(L, capture);  /* initialize caplistidx */
-  lua_getuservalue(L, 1);  /* initialize penvidx */
+  lua_getuservalue(L, 1);  /* initialize ktableidx */
   r = match(L, s, s + i, s + l, code, capture, ptop);
   if (r == NULL) {
     lua_pushnil(L);
@@ -1369,6 +1374,7 @@ static struct luaL_Reg metareg[] = {
   {"__gc", lp_gc},
   {"__len", lp_and},
   {"__div", lp_divcapture},
+  {"__mod", lp_acccapture},
   {"__unm", lp_not},
   {"__sub", lp_sub},
   {NULL, NULL}
diff --git a/makefile b/makefile
index 2b3c12e..2aff497 100644
--- a/makefile
+++ b/makefile
@@ -3,7 +3,7 @@ LUADIR = ./lua/
 
 # COPT = -O2 -DNDEBUG
 # COPT = -g
-COPT = -DLPEG_DEBUG -g
+COPT = -O0 -DLPEG_DEBUG -g
 
 CWARNS = -Wall -Wextra -pedantic \
         -Waggregate-return \
diff --git a/test.lua b/test.lua
index d31a69f..cd85b31 100755
--- a/test.lua
+++ b/test.lua
@@ -493,8 +493,8 @@ local function f_term (v1, op, v2, d)
 end
 
 G = m.P{ "Exp",
-  Exp = m.Cf(V"Factor" * m.Cg(FactorOp * V"Factor")^0, f_factor);
-  Factor = m.Cf(V"Term" * m.Cg(TermOp * V"Term")^0, f_term);
+  Exp = V"Factor" * (FactorOp * V"Factor" % f_factor)^0;
+  Factor = V"Term" * (TermOp * V"Term" % f_term)^0;
   Term = Number / tonumber  +  Open * V"Exp" * Close;
 }
 
@@ -866,6 +866,7 @@ print"+"
 -- accumulator capture
 function f (x) return x + 1 end
 assert(m.match(m.Cf(m.Cc(0) * m.C(1)^0, f), "alo alo") == 7)
+assert(m.match(m.Cc(0) * (m.C(1) % f)^0, "alo alo") == 7)
 
 t = {m.match(m.Cf(m.Cc(1,2,3), error), "")}
 checkeq(t, {1})
@@ -875,7 +876,7 @@ t = p:match("a=b;c=du;xux=yuy;")
 checkeq(t, {a="b", c="du", xux="yuy"})
 
 
--- errors in accumulator capture
+-- errors in fold capture
 
 -- no initial capture
 checkerr("no initial value", m.match, m.Cf(m.P(5), print), 'aaaaaa')
@@ -883,8 +884,14 @@ checkerr("no initial value", m.match, m.Cf(m.P(5), print), 'aaaaaa')
 checkerr("no initial value", m.match, m.Cf(m.P(500), print),
                                string.rep('a', 600))
 
--- nested capture produces no initial value
-checkerr("no initial value", m.match, m.Cf(m.P(1) / {}, print), "alo")
+
+-- errors in accumulator capture
+
+-- no initial capture
+checkerr("no previous value", m.match, m.P(5) % print, 'aaaaaa')
+-- no initial capture (very long match forces fold to be a pair open-close)
+checkerr("no previous value", m.match, m.P(500) % print,
+                               string.rep('a', 600))
 
 
 -- tests for loop checker