Get rid of Mail_mimeDecode package dependency (#1490416)

Aleksander Machniak

2015-06-27 f7427f151ef5020cd76d6005cc8b471763702f59

Get rid of Mail_mimeDecode package dependency (#1490416)

 CHANGELOG

@@ -1,6 +1,7 @@
CHANGELOG Roundcube Webmail
===========================

- Get rid of Mail_mimeDecode package dependency (#1490416)
- Add --config and --type options to moduserprefs.sh script (#1490051)
- Implemented memcache_debug and apc_debug options
- Installer: Remove system() function use (#1490139)

 INSTALL

@@ -18,7 +18,6 @@
   - OpenSSL, Fileinfo, Mcrypt, mbstring (optional)
* PEAR packages distributed with Roundcube or external:
   - Mail_Mime 1.8.1 or newer
   - Mail_mimeDecode 1.5.5 or newer
   - Net_SMTP (latest from https://github.com/pear/Net_SMTP/)
   - Net_IDNA2 0.1.1 or newer
   - Auth_SASL 1.0.6 or newer

 composer.json-dist

@@ -24,7 +24,6 @@
        "pear-pear.php.net/net_sieve": ">=1.3.2",
        "pear/mail_mime": ">=1.8.9",
        "pear/net_smtp": "dev-master",
        "pear/mail_mime-decode": ">=1.5.5",
        "patchwork/utf8": "1.2.x"
    },
    "require-dev": {

 installer/check.php

@@ -34,7 +34,6 @@
    'Net_SMTP'  => 'pear.php.net',
    'Net_IDNA2' => 'pear.php.net',
    'Mail_mime' => 'pear.php.net',
    'Mail_mimeDecode' => 'pear.php.net',
);

$optional_libs = array(
@@ -82,7 +81,6 @@
    'PEAR'      => 'http://pear.php.net',
    'Net_SMTP'  => 'http://pear.php.net/package/Net_SMTP',
    'Mail_mime' => 'http://pear.php.net/package/Mail_mime',
    'Mail_mimeDecode' => 'http://pear.php.net/package/Mail_mimeDecode',
    'Net_IDNA2' => 'http://pear.php.net/package/Net_IDNA2',
    'Net_LDAP3' => 'https://git.kolab.org/diffusion/PNL',
);

 program/lib/Roundcube/README.md

@@ -23,7 +23,6 @@
one or multiple of the following [PEAR][pear] libraries:

- Mail_Mime 1.8.1 or newer
- Mail_mimeDecode 1.5.5 or newer
- Net_SMTP (latest from https://github.com/pear/Net_SMTP/)
- Net_IDNA2 0.1.1 or newer
- Auth_SASL 1.0.6 or newer

 program/lib/Roundcube/rcube_imap.php

@@ -1756,10 +1756,10 @@

    /**
     * Fetch message headers and body structure from the IMAP server and build
     * an object structure similar to the one generated by PEAR::Mail_mimeDecode
     * an object structure.
     *
     * @param int     $uid      Message UID to fetch
     * @param string  $folder   Folder to read from
     * @param int    $uid    Message UID to fetch
     * @param string $folder Folder to read from
     *
     * @return object rcube_message_header Message data
     */
@@ -1838,8 +1838,8 @@
                $structure[1] = $m[2];
            }
            else {
                // Try to parse the message using Mail_mimeDecode package
                // We need a better solution, Mail_mimeDecode parses message
                // Try to parse the message using rcube_mime_decode.
                // We need a better solution, it parses message
                // in memory, which wouldn't work for very big messages,
                // (it uses up to 10x more memory than the message size)
                // it's also buggy and not actively developed

 program/lib/Roundcube/rcube_mime.php

@@ -61,61 +61,24 @@
     * Parse the given raw message source and return a structure
     * of rcube_message_part objects.
     *
     * It makes use of the PEAR:Mail_mimeDecode library
     * It makes use of the rcube_mime_decode library
     *
     * @param string  The message source
     * @param string $raw_body The message source
     *
     * @return object rcube_message_part The message structure
     */
    public static function parse_message($raw_body)
    {
        $mime = new Mail_mimeDecode($raw_body);
        $struct = $mime->decode(array('include_bodies' => true, 'decode_bodies' => true));
        return self::structure_part($struct);
    }
        $conf = array(
            'include_bodies'  => true,
            'decode_bodies'   => true,
            'decode_headers'  => false,
            'default_charset' => self::get_charset(),
        );

    /**
     * Recursive method to convert a Mail_mimeDecode part into a rcube_message_part object
     *
     * @param object  A message part struct
     * @param int     Part count
     * @param string  Parent MIME ID
     *
     * @return object rcube_message_part
     */
    private static function structure_part($part, $count=0, $parent='')
    {
        $struct = new rcube_message_part;
        $struct->mime_id          = $part->mime_id ?: (empty($parent) ? (string)$count : "$parent.$count");
        $struct->headers          = $part->headers;
        $struct->mimetype         = $part->ctype_primary . '/' . $part->ctype_secondary;
        $struct->ctype_primary    = $part->ctype_primary;
        $struct->ctype_secondary  = $part->ctype_secondary;
        $struct->ctype_parameters = $part->ctype_parameters;
        $mime = new rcube_mime_decode($conf);

        if ($part->headers['content-transfer-encoding']) {
            $struct->encoding = $part->headers['content-transfer-encoding'];
        }

        if ($part->ctype_parameters['charset']) {
            $struct->charset = $part->ctype_parameters['charset'];
        }

        $part_charset = $struct->charset ?: self::get_charset();

        // determine filename
        if (($filename = $part->d_parameters['filename']) || ($filename = $part->ctype_parameters['name'])) {
            $struct->filename = rcube_mime::decode_mime_string($filename, $part_charset);
        }

        $struct->body        = $part->body;
        $struct->size        = strlen($part->body);
        $struct->disposition = $part->disposition;

        foreach ((array)$part->parts as $child_part) {
            $struct->parts[] = self::structure_part($child_part, ++$count, $struct->mime_id);
        }

        return $struct;
        return $mime->decode($raw_body);
    }

    /**

 program/lib/Roundcube/rcube_mime_decode.php

New file
@@ -0,0 +1,397 @@
<?php

/**
 +-----------------------------------------------------------------------+
 | This file is part of the Roundcube Webmail client                     |
 | Copyright (C) 2005-2015, The Roundcube Dev Team                       |
 | Copyright (C) 2011-2015, Kolab Systems AG                             |
 |                                                                       |
 | Licensed under the GNU General Public License version 3 or            |
 | any later version with exceptions for skins & plugins.                |
 | See the README file for a full license statement.                     |
 |                                                                       |
 | PURPOSE:                                                              |
 |   MIME message parsing utilities derived from Mail_mimeDecode         |
 +-----------------------------------------------------------------------+
 | Author: Thomas Bruederli <roundcube@gmail.com>                        |
 | Author: Aleksander Machniak <alec@alec.pl>                            |
 | Author: Richard Heyes <richard@phpguru.org>                           |
 +-----------------------------------------------------------------------+
*/

/**
 * Class for parsing MIME messages
 *
 * @package    Framework
 * @subpackage Storage
 * @author     Aleksander Machniak <alec@alec.pl>
 */
class rcube_mime_decode
{
    /**
     * Class configuration parameters.
     *
     * @var array
     */
    protected $params = array(
        'include_bodies'  => true,
        'decode_bodies'   => true,
        'decode_headers'  => true,
        'crlf'            => "\r\n",
        'default_charset' => RCUBE_CHARSET,
    );


    /**
     * Constructor.
     *
     * Sets up the object, initialise the variables, and splits and
     * stores the header and body of the input.
     *
     * @param array $params An array of various parameters that determine
     *                       various things:
     *              include_bodies - Whether to include the body in the returned
     *                               object.
     *              decode_bodies  - Whether to decode the bodies
     *                               of the parts. (Transfer encoding)
     *              decode_headers - Whether to decode headers
     *              crlf           - CRLF type to use (CRLF/LF/CR)
     */
    public function __construct($params = array())
    {
        if (!empty($params)) {
            $this->params = array_merge($this->params, (array) $params);
        }
    }

    /**
     * Performs the decoding process.
     *
     * @param string $input The input to decode
     *
     * @return object|bool Decoded results or False on failure
     */
    public function decode($input)
    {
        list($header, $body) = $this->splitBodyHeader($input);

        // @TODO: Since this is a part of Roundcube Framework
        // we should return rcube_message_part structure

        if ($struct = $this->do_decode($header, $body)) {
            $struct = $this->structure_part($struct);
        }

        return $struct;
    }

    /**
     * Performs the decoding. Decodes the body string passed to it
     * If it finds certain content-types it will call itself in a
     * recursive fashion
     *
     * @param string $headers       Header section
     * @param string $body          Body section
     * @param string $default_ctype Default content type
     *
     * @return object|bool Decoded results or False on error
     */
    protected function do_decode($headers, $body, $default_ctype = 'text/plain')
    {
        $return  = new stdClass;
        $headers = $this->parseHeaders($headers);

        while (list($key, $value) = each($headers)) {
            $header_name = strtolower($value['name']);

            if (isset($return->headers[$header_name]) && !is_array($return->headers[$header_name])) {
                $return->headers[$header_name]   = array($return->headers[$header_name]);
                $return->headers[$header_name][] = $value['value'];
            }
            else if (isset($return->headers[$header_name])) {
                $return->headers[$header_name][] = $value['value'];
            }
            else {
                $return->headers[$header_name] = $value['value'];
            }

            switch ($header_name) {
            case 'content-type':
                $content_type = $this->parseHeaderValue($value['value']);

                if (preg_match('/([0-9a-z+.-]+)\/([0-9a-z+.-]+)/i', $content_type['value'], $regs)) {
                    $return->ctype_primary   = $regs[1];
                    $return->ctype_secondary = $regs[2];
                }

                if (isset($content_type['other'])) {
                    while (list($p_name, $p_value) = each($content_type['other'])) {
                        $return->ctype_parameters[$p_name] = $p_value;
                    }
                }

                break;

            case 'content-disposition';
                $content_disposition = $this->parseHeaderValue($value['value']);
                $return->disposition = $content_disposition['value'];

                if (isset($content_disposition['other'])) {
                    while (list($p_name, $p_value) = each($content_disposition['other'])) {
                        $return->d_parameters[$p_name] = $p_value;
                    }
                }

                break;

            case 'content-transfer-encoding':
                $content_transfer_encoding = $this->parseHeaderValue($value['value']);
                break;
            }
        }

        if (isset($content_type)) {
            $ctype = strtolower($content_type['value']);

            switch ($ctype) {
            case 'text/plain':
                $encoding = isset($content_transfer_encoding) ? $content_transfer_encoding['value'] : '7bit';

                if ($this->params['include_bodies']) {
                    $return->body = $this->params['decode_bodies'] ? rcube_mime::decode($body, $encoding) : $body;
                }

                break;

            case 'text/html':
                $encoding = isset($content_transfer_encoding) ? $content_transfer_encoding['value'] : '7bit';

                if ($this->params['include_bodies']) {
                    $return->body = $this->params['decode_bodies'] ? rcube_mime::decode($body, $encoding) : $body;
                }

                break;

            case 'multipart/digest':
            case 'multipart/alternative':
            case 'multipart/related':
            case 'multipart/mixed':
                if (!isset($content_type['other']['boundary'])) {
                    return false;
                }

                $default_ctype = $ctype === 'multipart/digest' ? 'message/rfc822' : 'text/plain';
                $parts         = $this->boundarySplit($body, $content_type['other']['boundary']);

                for ($i = 0; $i < count($parts); $i++) {
                    list($part_header, $part_body) = $this->splitBodyHeader($parts[$i]);
                    $return->parts[] = $this->do_decode($part_header, $part_body, $default_ctype);
                }

                break;

            case 'message/rfc822':
                $obj = new rcube_mime_decode($this->params);
                $return->parts[] = $obj->decode($body);
                unset($obj);
                break;

            default:
                if ($this->params['include_bodies']) {
                    $return->body = $this->params['decode_bodies'] ? rcube_mime::decode($body, $content_transfer_encoding['value']) : $body;
                }

                break;
            }
        }
        else {
            $ctype = explode('/', $default_ctype);
            $return->ctype_primary   = $ctype[0];
            $return->ctype_secondary = $ctype[1];

            if ($this->params['include_bodies']) {
                $return->body = $this->params['decode_bodies'] ? rcube_mime::decode($body) : $body;
            }
        }

        return $return;
    }

    /**
     * Given a string containing a header and body
     * section, this function will split them (at the first
     * blank line) and return them.
     *
     * @param string $input Input to split apart
     *
     * @return array Contains header and body section
     */
    protected function splitBodyHeader($input)
    {
        $pos = strpos($input, $this->params['crlf'] . $this->params['crlf']);
        if ($pos === false) {
            return false;
        }

        $crlf_len = strlen($this->params['crlf']);
        $header   = substr($input, 0, $pos);
        $body     = substr($input, $pos + 2 * $crlf_len);

        if (substr_compare($body, $this->params['crlf'], -$crlf_len) === 0) {
            $body = substr($body, 0, -$crlf_len);
        }

        return array($header, $body);
    }

    /**
     * Parse headers given in $input and return as assoc array.
     *
     * @param string $input Headers to parse
     *
     * @return array Contains parsed headers
     */
    protected function parseHeaders($input)
    {
        if ($input !== '') {
            // Unfold the input
            $input   = preg_replace('/' . $this->params['crlf'] . "(\t| )/", ' ', $input);
            $headers = explode($this->params['crlf'], trim($input));

            foreach ($headers as $value) {
                $hdr_name  = substr($value, 0, $pos = strpos($value, ':'));
                $hdr_value = substr($value, $pos+1);

                if ($hdr_value[0] == ' ') {
                    $hdr_value = substr($hdr_value, 1);
                }

                $return[] = array(
                    'name'  => $hdr_name,
                    'value' => $this->params['decode_headers'] ? $this->decodeHeader($hdr_value) : $hdr_value,
                );
            }
        }
        else {
            $return = array();
        }

        return $return;
    }

    /**
     * Function to parse a header value, extract first part, and any secondary
     * parts (after ;) This function is not as robust as it could be.
     * Eg. header comments in the wrong place will probably break it.
     *
     * @param string $input Header value to parse
     *
     * @return array Contains parsed result
     */
    protected function parseHeaderValue($input)
    {
        $parts = preg_split('/;\s*/', $input);

        if (!empty($parts)) {
            $return['value'] = trim($parts[0]);

            for ($n = 1; $n < count($parts); $n++) {
                if (preg_match_all('/(([[:alnum:]]+)="?([^"]*)"?\s?;?)+/i', $parts[$n], $matches)) {
                    for ($i = 0; $i < count($matches[2]); $i++) {
                        $return['other'][strtolower($matches[2][$i])] = $matches[3][$i];
                    }
                }
            }
        }
        else {
            $return['value'] = trim($input);
        }

        return $return;
    }

    /**
     * This function splits the input based on the given boundary
     *
     * @param string $input    Input to parse
     * @param string $boundary Boundary
     *
     * @return array Contains array of resulting mime parts
     */
    protected function boundarySplit($input, $boundary)
    {
        $tmp = explode('--' . $boundary, $input);

        for ($i = 1; $i < count($tmp)-1; $i++) {
            $parts[] = $tmp[$i];
        }

        return $parts;
    }

    /**
     * Given a header, this function will decode it according to RFC2047.
     * Probably not *exactly* conformant, but it does pass all the given
     * examples (in RFC2047).
     *
     * @param string $input Input header value to decode
     *
     * @return string Decoded header value
     */
    protected function decodeHeader($input)
    {
        return rcube_mime::decode_mime_string($input, $this->params['default_charset']);
    }

    /**
     * Recursive method to convert a rcube_mime_decode structure
     * into a rcube_message_part object.
     *
     * @param object $part   A message part struct
     * @param int    $count  Part count
     * @param string $parent Parent MIME ID
     *
     * @return object rcube_message_part
     * @see self::decode()
     */
    protected function structure_part($part, $count = 0, $parent = '')
    {
        $struct = new rcube_message_part;
        $struct->mime_id          = $part->mime_id ?: (empty($parent) ? (string)$count : "$parent.$count");
        $struct->headers          = $part->headers;
        $struct->mimetype         = $part->ctype_primary . '/' . $part->ctype_secondary;
        $struct->ctype_primary    = $part->ctype_primary;
        $struct->ctype_secondary  = $part->ctype_secondary;
        $struct->ctype_parameters = $part->ctype_parameters;

        if ($part->headers['content-transfer-encoding']) {
            $struct->encoding = $part->headers['content-transfer-encoding'];
        }

        if ($part->ctype_parameters['charset']) {
            $struct->charset = $part->ctype_parameters['charset'];
        }

        $part_charset = $struct->charset ?: $this->params['default_charset'];

        // determine filename
        if (($filename = $part->d_parameters['filename']) || ($filename = $part->ctype_parameters['name'])) {
            if (!$this->params['decode_headers']) {
                $filename = $this->decodeHeader($filename);
            }

            $struct->filename = $filename;
        }

        $struct->body        = $part->body;
        $struct->size        = strlen($part->body);
        $struct->disposition = $part->disposition;

        $count = 0;
        foreach ((array)$part->parts as $child_part) {
            $struct->parts[] = $this->structure_part($child_part, ++$count, $struct->mime_id);
        }

        return $struct;
    }
}

 program/lib/Roundcube/rcube_storage.php

@@ -405,7 +405,7 @@

    /**
     * Fetch message headers and body structure from the server and build
     * an object structure similar to the one generated by PEAR::Mail_mimeDecode
     * an object structure.
     *
     * @param int     $uid     Message UID to fetch
     * @param string  $folder  Folder to read from

 tests/Framework/Mime.php

@@ -231,4 +231,36 @@
        }
    }

    /**
     * Test parse_message()
     */
    function test_parse_message()
    {
        $file   = file_get_contents(__DIR__ . '/../src/html.msg');
        $result = rcube_mime::parse_message($file);

        $this->assertInstanceOf('rcube_message_part', $result);
        $this->assertSame('multipart/alternative', $result->mimetype);
        $this->assertSame('1.0', $result->headers['mime-version']);
        $this->assertSame('=_68eeaf4ab95b5312965e45c33362338e', $result->ctype_parameters['boundary']);
        $this->assertSame('1',              $result->parts[0]->mime_id);
        $this->assertSame(12,               $result->parts[0]->size);
        $this->assertSame('text/plain',     $result->parts[0]->mimetype);
        $this->assertSame("this is test",   $result->parts[0]->body);
        $this->assertSame('2',              $result->parts[1]->mime_id);
        $this->assertSame(0,                $result->parts[1]->size);
        $this->assertSame('multipart/related', $result->parts[1]->mimetype);
        $this->assertCount(2,               $result->parts[1]->parts);
        $this->assertSame('2.1',            $result->parts[1]->parts[0]->mime_id);
        $this->assertSame(257,              $result->parts[1]->parts[0]->size);
        $this->assertSame('text/html',      $result->parts[1]->parts[0]->mimetype);
        $this->assertSame('UTF-8',          $result->parts[1]->parts[0]->charset);
        $this->assertRegExp('/<html>/',     $result->parts[1]->parts[0]->body);
        $this->assertSame('2.2',            $result->parts[1]->parts[1]->mime_id);
        $this->assertSame(793,              $result->parts[1]->parts[1]->size);
        $this->assertSame('image/jpeg',     $result->parts[1]->parts[1]->mimetype);
        $this->assertSame('base64',          $result->parts[1]->parts[1]->encoding);
        $this->assertSame('inline',          $result->parts[1]->parts[1]->disposition);
        $this->assertSame('photo-mini.jpg', $result->parts[1]->parts[1]->filename);
    }
}

 tests/src/html.msg

New file
@@ -0,0 +1,56 @@
MIME-Version: 1.0

Content-Type: multipart/alternative;

 boundary="=_68eeaf4ab95b5312965e45c33362338e"

Date: Sat, 27 Jun 2015 08:41:48 +0200

From: alec@domain.tld

To: alec@domain.tld

Subject: test html + image

Message-ID: <b490c541b3bf9d6c03a65b6da6e3152f@alec.pl>

User-Agent: Roundcube Webmail/1.2-git



--=_68eeaf4ab95b5312965e45c33362338e

Content-Transfer-Encoding: 7bit

Content-Type: text/plain; charset=US-ASCII



this is test

--=_68eeaf4ab95b5312965e45c33362338e

Content-Type: multipart/related;

 boundary="=_a9368338354ce1dee6d068bda60f3cc4"



--=_a9368338354ce1dee6d068bda60f3cc4

Content-Transfer-Encoding: quoted-printable

Content-Type: text/html; charset=UTF-8



<html><head><meta http-equiv=3D"Content-Type" content=3D"text/html; charset=

=3DUTF-8" /></head><body style=3D'font-size: 10pt'>

<p>this is test</p>

<p><img src=3D"cid:1435387308558e45acc0292830140312@domain.tld" alt=3D"" width=

=3D"24" height=3D"24" /></p>

</body></html>



--=_a9368338354ce1dee6d068bda60f3cc4

Content-Transfer-Encoding: base64

Content-ID: <1435387308558e45acc0292830140312@domain.tld>

Content-Type: image/jpeg;

 name=photo-mini.jpg

Content-Disposition: inline;

 filename=photo-mini.jpg;

 size=793



/9j/4AAQSkZJRgABAQEASABIAAD/2wBDAAoHBwkHBgoJCAkLCwoMDxkQDw4ODx4WFxIZJCAmJSMg

IyIoLTkwKCo2KyIjMkQyNjs9QEBAJjBGS0U+Sjk/QD3/2wBDAQsLCw8NDx0QEB09KSMpPT09PT09

PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT09PT3/wgARCAAYABgDAREA

AhEBAxEB/8QAFgABAQEAAAAAAAAAAAAAAAAABAUD/8QAFgEBAQEAAAAAAAAAAAAAAAAAAwIB/9oA

DAMBAAIQAxAAAAGjlGNp6hVmsTcdm6jmSrLH/8QAGxAAAwACAwAAAAAAAAAAAAAAAQIDABESISP/

2gAIAQEAAQUCqOQaA0GJlUFs7c00q1bznQK1oFp//8QAGREAAwEBAQAAAAAAAAAAAAAAAAERAgMS

/9oACAEDAQE/Acqs8ZJGYIdJROC2h6rp/8QAGhEAAwADAQAAAAAAAAAAAAAAAAERAhAxEv/aAAgB

AgEBPwFuHplqHrHh0aYsYj//xAAgEAABBAEEAwAAAAAAAAAAAAAAAQIREjEQIjIzQVFh/9oACAEB

AAY/Aq+zzI62UlBKuqp2YH/dIRjpNvI//8QAHBABAQEAAwADAAAAAAAAAAAAAREAITFBUXGR/9oA

CAEBAAE/IZNdtXiPkOK/UY5yPZdCtQFD3GF3S/mg+36zcxM67zxjhyXf/9oADAMBAAIAAwAAABAU

fgX/AP/EABgRAQEBAQEAAAAAAAAAAAAAAAERACEx/9oACAEDAQE/EONrEmeHFVDDY4AT2624BQmP

b//EABkRAQEBAQEBAAAAAAAAAAAAAAERADEQQf/aAAgBAgEBPxCC5Bt3R5fpm1cnGUzLHf/EAB8Q

AQEAAgIDAAMAAAAAAAAAAAERACExQVFxgZGhsf/aAAgBAQABPxAtWaL2BtT9H3BdmhtH1x8w1QFB

woc5u4oJcpqPrBACtyKVdfzHhmnXZYygBKAJFrNTveSICmoJy9LdauUhKVIN8L41+c//2Q==

--=_a9368338354ce1dee6d068bda60f3cc4--



--=_68eeaf4ab95b5312965e45c33362338e--