Added support for date and index extensions (RFC5260) in script parser Scri...

Aleksander Machniak

2013-05-12 ed837a3544b8c157b00f520e0c716304abedd519

Added support for date and index extensions (RFC5260) in script parser
Script parser code improvements

 plugins/managesieve/Changelog

@@ -1,4 +1,5 @@
- Add vacation-seconds extension support (RFC 6131)
- Several script parser code improvements

* version 6.2 [2013-02-17]
-----------------------------------------------------------

 plugins/managesieve/lib/Roundcube/rcube_sieve_script.php

@@ -27,24 +27,26 @@
    private $vars = array();        // "global" variables
    private $prefix = '';           // script header (comments)
    private $supported = array(     // Sieve extensions supported by class
        'fileinto',                 // RFC5228
        'envelope',                 // RFC5228
        'reject',                   // RFC5429
        'ereject',                  // RFC5429
        'body',                     // RFC5173
        'copy',                     // RFC3894
        'vacation',                 // RFC5230
        'vacation-seconds',         // RFC6131
        'relational',               // RFC3431
        'regex',                    // draft-ietf-sieve-regex-01
        'date',                     // RFC5260
        'enotify',                  // RFC5435
        'envelope',                 // RFC5228
        'ereject',                  // RFC5429
        'fileinto',                 // RFC5228
        'imapflags',                // draft-melnikov-sieve-imapflags-06
        'imap4flags',               // RFC5232
        'include',                  // draft-ietf-sieve-include-12
        'variables',                // RFC5229
        'body',                     // RFC5173
        'subaddress',               // RFC5233
        'enotify',                  // RFC5435
        'index',                    // RFC5260
        'notify',                   // draft-ietf-sieve-notify-00
        // @TODO: spamtest+virustest, mailbox, date
        'regex',                    // draft-ietf-sieve-regex-01
        'reject',                   // RFC5429
        'relational',               // RFC3431
        'subaddress',               // RFC5233
        'vacation',                 // RFC5230
        'vacation-seconds',         // RFC6131
        'variables',                // RFC5229
        // @TODO: spamtest+virustest, mailbox
    );

    /**
@@ -238,24 +240,8 @@
                        $tests[$i] .= ($test['not'] ? 'not ' : '');
                        $tests[$i] .= 'header';

                        if (!empty($test['type'])) {
                            // relational operator + comparator
                            if (preg_match('/^(value|count)-([gteqnl]{2})/', $test['type'], $m)) {
                                array_push($exts, 'relational');
                                array_push($exts, 'comparator-i;ascii-numeric');

                                $tests[$i] .= ' :' . $m[1] . ' "' . $m[2] . '" :comparator "i;ascii-numeric"';
                            }
                            else {
                                $this->add_comparator($test, $tests[$i], $exts);

                                if ($test['type'] == 'regex') {
                                    array_push($exts, 'regex');
                                }

                                $tests[$i] .= ' :' . $test['type'];
                            }
                        }
                        $this->add_index($test, $tests[$i], $exts);
                        $this->add_operator($test, $tests[$i], $exts);

                        $tests[$i] .= ' ' . self::escape_string($test['arg1']);
                        $tests[$i] .= ' ' . self::escape_string($test['arg2']);
@@ -270,6 +256,10 @@
                        $tests[$i] .= ($test['not'] ? 'not ' : '');
                        $tests[$i] .= $test['test'];

                        if ($test['test'] != 'envelope') {
                            $this->add_index($test, $tests[$i], $exts);
                        }

                        if (!empty($test['part'])) {
                            $tests[$i] .= ' :' . $test['part'];
                            if ($test['part'] == 'user' || $test['part'] == 'detail') {
@@ -277,14 +267,7 @@
                            }
                        }

                        $this->add_comparator($test, $tests[$i], $exts);

                        if (!empty($test['type'])) {
                            if ($test['type'] == 'regex') {
                                array_push($exts, 'regex');
                            }
                            $tests[$i] .= ' :' . $test['type'];
                        }
                        $this->add_operator($test, $tests[$i], $exts);

                        $tests[$i] .= ' ' . self::escape_string($test['arg1']);
                        $tests[$i] .= ' ' . self::escape_string($test['arg2']);
@@ -295,8 +278,6 @@

                        $tests[$i] .= ($test['not'] ? 'not ' : '') . 'body';

                        $this->add_comparator($test, $tests[$i], $exts);

                        if (!empty($test['part'])) {
                            $tests[$i] .= ' :' . $test['part'];

@@ -305,14 +286,35 @@
                            }
                        }

                        if (!empty($test['type'])) {
                            if ($test['type'] == 'regex') {
                                array_push($exts, 'regex');
                            }
                            $tests[$i] .= ' :' . $test['type'];
                        }
                        $this->add_operator($test, $tests[$i], $exts);

                        $tests[$i] .= ' ' . self::escape_string($test['arg']);
                        break;

                    case 'date':
                    case 'currentdate':
                        array_push($exts, 'date');

                        $tests[$i] .= ($test['not'] ? 'not ' : '') . $test['test'];

                        $this->add_index($test, $tests[$i], $exts);

                        if (!empty($test['originalzone']) && $test['test'] == 'date') {
                            $tests[$i] .= ' :originalzone';
                        }
                        else if (!empty($test['zone'])) {
                            $tests[$i] .= ' :zone ' . self::escape_string($test['zone']);
                        }

                        $this->add_operator($test, $tests[$i], $exts);

                        if ($test['test'] == 'date') {
                            $tests[$i] .= ' ' . self::escape_string($test['header']);
                        }

                        $tests[$i] .= ' ' . self::escape_string($test['part']);
                        $tests[$i] .= ' ' . self::escape_string($test['arg']);

                        break;
                    }
                    $i++;
@@ -489,6 +491,8 @@
                unset($exts[$key]);
            }

            sort($exts); // for convenience use always the same order

            $output = 'require ["' . implode('","', $exts) . "\"];\n" . $output;
        }

@@ -652,86 +656,85 @@
                break;

            case 'size':
                $size = array('test' => 'size', 'not'  => $not);
                $test = array('test' => 'size', 'not'  => $not);

                $test['arg'] = array_pop($tokens);

                for ($i=0, $len=count($tokens); $i<$len; $i++) {
                    if (!is_array($tokens[$i])
                        && preg_match('/^:(under|over)$/i', $tokens[$i])
                    ) {
                        $size['type'] = strtolower(substr($tokens[$i], 1));
                    }
                    else {
                        $size['arg'] = $tokens[$i];
                        $test['type'] = strtolower(substr($tokens[$i], 1));
                    }
                }

                $tests[] = $size;
                $tests[] = $test;
                break;

            case 'header':
                $header = array('test' => 'header', 'not' => $not, 'arg1' => '', 'arg2' => '');
                for ($i=0, $len=count($tokens); $i<$len; $i++) {
                    if (!is_array($tokens[$i]) && preg_match('/^:comparator$/i', $tokens[$i])) {
                        $header['comparator'] = $tokens[++$i];
                    }
                    else if (!is_array($tokens[$i]) && preg_match('/^:(count|value)$/i', $tokens[$i])) {
                        $header['type'] = strtolower(substr($tokens[$i], 1)) . '-' . $tokens[++$i];
                    }
                    else if (!is_array($tokens[$i]) && preg_match('/^:(is|contains|matches|regex)$/i', $tokens[$i])) {
                        $header['type'] = strtolower(substr($tokens[$i], 1));
                    }
                    else {
                        $header['arg1'] = $header['arg2'];
                        $header['arg2'] = $tokens[$i];
                    }
                }

                $tests[] = $header;
                break;

            case 'address':
            case 'envelope':
                $header = array('test' => $token, 'not' => $not, 'arg1' => '', 'arg2' => '');
                for ($i=0, $len=count($tokens); $i<$len; $i++) {
                    if (!is_array($tokens[$i]) && preg_match('/^:comparator$/i', $tokens[$i])) {
                        $header['comparator'] = $tokens[++$i];
                    }
                    else if (!is_array($tokens[$i]) && preg_match('/^:(is|contains|matches|regex)$/i', $tokens[$i])) {
                        $header['type'] = strtolower(substr($tokens[$i], 1));
                    }
                    else if (!is_array($tokens[$i]) && preg_match('/^:(localpart|domain|all|user|detail)$/i', $tokens[$i])) {
                        $header['part'] = strtolower(substr($tokens[$i], 1));
                    }
                    else {
                        $header['arg1'] = $header['arg2'];
                        $header['arg2'] = $tokens[$i];
                $test = array('test' => $token, 'not' => $not);

                $test['arg2'] = array_pop($tokens);
                $test['arg1'] = array_pop($tokens);

                $test += $this->test_tokens($tokens);

                if ($token != 'header' && !empty($tokens)) {
                    for ($i=0, $len=count($tokens); $i<$len; $i++) {
                        if (!is_array($tokens[$i]) && preg_match('/^:(localpart|domain|all|user|detail)$/i', $tokens[$i])) {
                            $test['part'] = strtolower(substr($tokens[$i], 1));
                        }
                    }
                }

                $tests[] = $header;
                $tests[] = $test;
                break;

            case 'body':
                $header = array('test' => 'body', 'not' => $not, 'arg' => '');
                for ($i=0, $len=count($tokens); $i<$len; $i++) {
                    if (!is_array($tokens[$i]) && preg_match('/^:comparator$/i', $tokens[$i])) {
                        $header['comparator'] = $tokens[++$i];
                    }
                    else if (!is_array($tokens[$i]) && preg_match('/^:(is|contains|matches|regex)$/i', $tokens[$i])) {
                        $header['type'] = strtolower(substr($tokens[$i], 1));
                    }
                    else if (!is_array($tokens[$i]) && preg_match('/^:(raw|content|text)$/i', $tokens[$i])) {
                        $header['part'] = strtolower(substr($tokens[$i], 1));
                $test = array('test' => 'body', 'not' => $not);

                        if ($header['part'] == 'content') {
                            $header['content'] = $tokens[++$i];
                $test['arg'] = array_pop($tokens);

                $test += $this->test_tokens($tokens);

                for ($i=0, $len=count($tokens); $i<$len; $i++) {
                    if (!is_array($tokens[$i]) && preg_match('/^:(raw|content|text)$/i', $tokens[$i])) {
                        $test['part'] = strtolower(substr($tokens[$i], 1));

                        if ($test['part'] == 'content') {
                            $test['content'] = $tokens[++$i];
                        }
                    }
                    else {
                        $header['arg'] = $tokens[$i];
                    }
                }

                $tests[] = $header;
                $tests[] = $test;
                break;

            case 'date':
            case 'currentdate':
                $test = array('test' => $token, 'not' => $not);

                $test['arg']  = array_pop($tokens);
                $test['part'] = array_pop($tokens);

                if ($token == 'date') {
                    $test['header']  = array_pop($tokens);
                }

                $test += $this->test_tokens($tokens);

                for ($i=0, $len=count($tokens); $i<$len; $i++) {
                    if (!is_array($tokens[$i]) && preg_match('/^:zone$/i', $tokens[$i])) {
                        $test['zone'] = $tokens[++$i];
                    }
                    else if (!is_array($tokens[$i]) && preg_match('/^:originalzone$/i', $tokens[$i])) {
                        $test['originalzone'] = true;
                    }
                }

                $tests[] = $test;
                break;

            case 'exists':
@@ -783,15 +786,9 @@
        $result = null;

        while (strlen($content)) {
            $tokens = self::tokenize($content, true);
            $tokens    = self::tokenize($content, true);
            $separator = array_pop($tokens);

            if (!empty($tokens)) {
                $token = array_shift($tokens);
            }
            else {
                $token = $separator;
            }
            $token     = !empty($tokens) ? array_shift($tokens) : $separator;

            switch ($token) {
            case 'discard':
@@ -802,113 +799,78 @@

            case 'fileinto':
            case 'redirect':
                $copy   = false;
                $target = '';
                $action  = array('type' => $token, 'target' => array_pop($tokens));
                $args    = array('copy');
                $action += $this->action_arguments($tokens, $args);

                for ($i=0, $len=count($tokens); $i<$len; $i++) {
                    if (strtolower($tokens[$i]) == ':copy') {
                        $copy = true;
                    }
                    else {
                        $target = $tokens[$i];
                    }
                }
                $result[] = $action;
                break;

                $result[] = array('type' => $token, 'copy' => $copy,
                    'target' => $target);
            case 'vacation':
                $action  = array('type' => 'vacation', 'reason' => array_pop($tokens));
                $args    = array('mime');
                $vargs   = array('seconds', 'days', 'addresses', 'subject', 'handle', 'from');
                $action += $this->action_arguments($tokens, $args, $vargs);

                $result[] = $action;
                break;

            case 'reject':
            case 'ereject':
                $result[] = array('type' => $token, 'target' => array_pop($tokens));
                break;

            case 'vacation':
                $vacation = array('type' => 'vacation', 'reason' => array_pop($tokens));

                for ($i=0, $len=count($tokens); $i<$len; $i++) {
                    $tok = strtolower($tokens[$i]);
                    if ($tok == ':mime') {
                        $vacation['mime'] = true;
                    }
                    else if ($tok[0] == ':') {
                        $vacation[substr($tok, 1)] = $tokens[++$i];
                    }
                }

                $result[] = $vacation;
                break;

            case 'setflag':
            case 'addflag':
            case 'removeflag':
                $result[] = array('type' => $token,
                    // Flags list: last token (skip optional variable)
                    'target' => $tokens[count($tokens)-1]
                );
                $result[] = array('type' => $token, 'target' => array_pop($tokens));
                break;

            case 'include':
                $include = array('type' => 'include', 'target' => array_pop($tokens));
                $action  = array('type' => 'include', 'target' => array_pop($tokens));
                $args    = array('once', 'optional', 'global', 'personal');
                $action += $this->action_arguments($tokens, $args);

                // Parameters: :once, :optional, :global, :personal
                for ($i=0, $len=count($tokens); $i<$len; $i++) {
                    $tok = strtolower($tokens[$i]);
                    if ($tok[0] == ':') {
                        $include[substr($tok, 1)] = true;
                    }
                }

                $result[] = $include;
                $result[] = $action;
                break;

            case 'set':
                $set = array('type' => 'set', 'value' => array_pop($tokens), 'name' => array_pop($tokens));
                $action  = array('type' => 'set', 'value' => array_pop($tokens), 'name' => array_pop($tokens));
                $args    = array('lower', 'upper', 'lowerfirst', 'upperfirst', 'quotewildcard', 'length');
                $action += $this->action_arguments($tokens, $args);

                // Parameters: :lower :upper :lowerfirst :upperfirst :quotewildcard :length
                for ($i=0, $len=count($tokens); $i<$len; $i++) {
                    $tok = strtolower($tokens[$i]);
                    if ($tok[0] == ':') {
                        $set[substr($tok, 1)] = true;
                    }
                }

                $result[] = $set;
                $result[] = $action;
                break;

            case 'require':
                // skip, will be build according to used commands
                // $result[] = array('type' => 'require', 'target' => $tokens);
                // $result[] = array('type' => 'require', 'target' => array_pop($tokens));
                break;

            case 'notify':
                $notify = array('type' => 'notify');
                $priorities = array(':high' => 1, ':normal' => 2, ':low' => 3);
                $action     = array('type' => 'notify');
                $priorities = array('high' => 1, 'normal' => 2, 'low' => 3);
                $vargs      = array('from', 'importance', 'options', 'message', 'method');
                $args       = array_keys($priorities);
                $action    += $this->action_arguments($tokens, $args, $vargs);

                // Parameters: :from, :importance, :options, :message
                //     additional (optional) :method parameter for notify extension
                for ($i=0, $len=count($tokens); $i<$len; $i++) {
                    $tok = strtolower($tokens[$i]);
                    if ($tok[0] == ':') {
                        // Here we support only 00 version of notify draft, there
                        // were a couple regressions in 00 to 04 changelog, we use
                        // the version used by Cyrus
                        if (isset($priorities[$tok])) {
                            $notify['importance'] = $priorities[$tok];
                // Here we support only 00 version of notify draft, there
                // were a couple regressions in 00 to 04 changelog, we use
                // the version used by Cyrus
                if (!isset($action['importance'])) {
                    foreach ($priorities as $key => $val) {
                        if (isset($action[$key])) {
                            $action['importance'] = $val;
                            unset($action[$key]);
                        }
                        else {
                            $notify[substr($tok, 1)] = $tokens[++$i];
                        }
                    }
                    else {
                        // unnamed parameter is a :method in enotify extension
                        $notify['method'] = $tokens[$i];
                    }
                }

                $method_components = parse_url($notify['method']);
                // unnamed parameter is a :method in enotify extension
                if (!isset($action['method'])) {
                    $action['method'] = array_pop($tokens);
                }

                $method_components = parse_url($action['method']);
                if ($method_components['scheme'] == 'mailto') {
                    $notify['address'] = $method_components['path'];
                    $action['address'] = $method_components['path'];
                    $method_params = array();
                    if (array_key_exists('query', $method_components)) {
                        parse_str($method_components['query'], $method_params);
@@ -918,10 +880,10 @@
                    if (ini_get('magic_quotes_gpc') || ini_get('magic_quotes_sybase')) {
                        array_map('stripslashes', $method_params);
                    }
                    $notify['body'] = (array_key_exists('body', $method_params)) ? $method_params['body'] : '';
                    $action['body'] = (array_key_exists('body', $method_params)) ? $method_params['body'] : '';
                }

                $result[] = $notify;
                $result[] = $action;
                break;

            }
@@ -934,7 +896,7 @@
    }

    /**
     *
     * Add comparator to the test
     */
    private function add_comparator($test, &$out, &$exts)
    {
@@ -957,6 +919,111 @@
    }

    /**
     * Add index argument to the test
     */
    private function add_index($test, &$out, &$exts)
    {
        if (!empty($test['index'])) {
            array_push($exts, 'index');
            $out .= ' :index ' . intval($test['index']) . ($test['last'] ? ' :last' : '');
        }
    }

    /**
     * Add operators to the test
     */
    private function add_operator($test, &$out, &$exts)
    {
        if (empty($test['type'])) {
            return;
        }

        // relational operator + comparator
        if (preg_match('/^(value|count)-([gteqnl]{2})/', $test['type'], $m)) {
            array_push($exts, 'relational');
            array_push($exts, 'comparator-i;ascii-numeric');

            $out .= ' :' . $m[1] . ' "' . $m[2] . '" :comparator "i;ascii-numeric"';
        }
        else {
            $this->add_comparator($test, $out, $exts);

            if ($test['type'] == 'regex') {
                array_push($exts, 'regex');
            }

            $out .= ' :' . $test['type'];
        }
    }

    /**
     * Extract test tokens
     */
    private function test_tokens(&$tokens)
    {
        $test   = array();
        $result = array();

        for ($i=0, $len=count($tokens); $i<$len; $i++) {
            if (!is_array($tokens[$i]) && preg_match('/^:comparator$/i', $tokens[$i])) {
                $test['comparator'] = $tokens[++$i];
            }
            else if (!is_array($tokens[$i]) && preg_match('/^:(count|value)$/i', $tokens[$i])) {
                $test['type'] = strtolower(substr($tokens[$i], 1)) . '-' . $tokens[++$i];
            }
            else if (!is_array($tokens[$i]) && preg_match('/^:(is|contains|matches|regex)$/i', $tokens[$i])) {
                $test['type'] = strtolower(substr($tokens[$i], 1));
            }
            else if (!is_array($tokens[$i]) && preg_match('/^:index$/i', $tokens[$i])) {
                $test['index'] = intval($tokens[++$i]);
                if ($tokens[$i+1] && preg_match('/^:last$/i', $tokens[$i+1])) {
                    $test['last'] = true;
                    $i++;
                }
           }
           else {
               $result[] = $tokens[$i];
           }
        }

        $tokens = $result;

        return $test;
    }

    /**
     * Extract action arguments
     */
    private function action_arguments(&$tokens, $bool_args, $val_args = array())
    {
        $action = array();
        $result = array();

        for ($i=0, $len=count($tokens); $i<$len; $i++) {
            $tok = $tokens[$i];
            if (!is_array($tok) && $tok[0] == ':') {
                $tok = strtolower(substr($tok, 1));
                if (in_array($tok, $bool_args)) {
                    $action[$tok] = true;
                }
                else if (in_array($tok, $val_args)) {
                    $action[$tok] = $tokens[++$i];
                }
                else {
                    $result[] = $tok;
                }
            }
            else {
                $result[] = $tok;
            }
        }

        $tokens = $result;

        return $action;
    }

    /**
     * Escape special chars into quoted string value or multi-line string
     * or list of strings
     *

 plugins/managesieve/tests/src/parser.out

@@ -1,4 +1,4 @@
require ["fileinto","reject","envelope"];
require ["envelope","fileinto","reject"];
# rule:[spam]
if header :contains "X-DSPAM-Result" "Spam"
{

 plugins/managesieve/tests/src/parser_date

New file
@@ -0,0 +1,21 @@
require ["comparator-i;ascii-numeric","date","fileinto","relational"];
# rule:[date]
if allof (date :originalzone :value "ge" :comparator "i;ascii-numeric" "date" "hour" "09")
{
    fileinto "urgent";
}
# rule:[date-weekday]
if date :is "received" "weekday" "0"
{
    fileinto "weekend";
}
# rule:[date-zone]
if date :zone "-0500" :value "gt" :comparator "i;ascii-numeric" "received" "iso8601" "2007-02-26T09:00:00-05:00"
{
    stop;
}
# rule:[currentdate]
if anyof (currentdate :is "weekday" "0", currentdate :value "lt" :comparator "i;ascii-numeric" "hour" "09", currentdate :value "ge" :comparator "i;ascii-numeric" "date" "2007-06-30")
{
    stop;
}

 plugins/managesieve/tests/src/parser_enotify_b

@@ -1,4 +1,4 @@
require ["envelope","variables","enotify"];
require ["enotify","envelope","variables"];
# rule:[from]
if envelope :all :matches "from" "*"
{

 plugins/managesieve/tests/src/parser_index

New file
@@ -0,0 +1,24 @@
require ["comparator-i;ascii-numeric","date","fileinto","index","relational"];
# rule:[index-header1]
if header :index 1 :last :contains "X-DSPAM-Result" "Spam"
{
    fileinto "Spam";
    stop;
}
# rule:[index-header2]
if header :index 2 :contains ["From","To"] "test@domain.tld"
{
    discard;
    stop;
}
# rule:[index-address]
if address :index 1 :all :is "From" "nagios@domain.tld"
{
    fileinto "domain.tld";
    stop;
}
# rule:[index-date]
if date :index 1 :last :zone "-0500" :value "gt" :comparator "i;ascii-numeric" "received" "iso8601" "2007-02-26T09:00:00-05:00"
{
    stop;
}

 plugins/managesieve/tests/src/parser_notify_b

@@ -1,4 +1,4 @@
require ["envelope","variables","notify"];
require ["envelope","notify","variables"];
# rule:[from]
if envelope :all :matches "from" "*"
{

 plugins/managesieve/tests/src/parser_relational

@@ -1,4 +1,4 @@
require ["relational","comparator-i;ascii-numeric"];
require ["comparator-i;ascii-numeric","relational"];
# rule:[redirect]
if header :value "ge" :comparator "i;ascii-numeric" "X-Spam-score" "14"
{

 plugins/managesieve/tests/src/parser_subaddress

@@ -1,4 +1,4 @@
require ["envelope","subaddress","fileinto"];
require ["envelope","fileinto","subaddress"];
if envelope :user "To" "postmaster"
{
    fileinto "postmaster";