Support contacts import from CSV file (#1486399)

Aleksander Machniak

2012-11-16 3833790db4dee8607b31c84f26eb0e95bae4c906

Support contacts import from CSV file (#1486399)

 CHANGELOG

@@ -1,6 +1,7 @@
CHANGELOG Roundcube Webmail
===========================

- Support contacts import from CSV file (#1486399)
- Improved keep-alive action. Now the interval is based on session_lifetime (#1488507)
- Added cross-task 'refresh' request for system state updates (#1488507)
- Renamed config options: keep_alive to refresh_interval, min_keep_alive to min_refresh_interval

 program/include/rcube_csv2vcard.php

New file
@@ -0,0 +1,351 @@
<?php

/*
 +-----------------------------------------------------------------------+
 | program/include/rcube_csv2vcard.php                                   |
 |                                                                       |
 | This file is part of the Roundcube Webmail client                     |
 | Copyright (C) 2008-2012, The Roundcube Dev Team                       |
 |                                                                       |
 | Licensed under the GNU General Public License version 3 or            |
 | any later version with exceptions for skins & plugins.                |
 | See the README file for a full license statement.                     |
 |                                                                       |
 | PURPOSE:                                                              |
 |   CSV to vCard data conversion                                        |
 +-----------------------------------------------------------------------+
 | Author: Aleksander Machniak <alec@alec.pl>                            |
 +-----------------------------------------------------------------------+
*/

/**
 * CSV to vCard data converter
 *
 * @package Roundcube Framework
 * @author  Aleksander Machniak <alec@alec.pl>
 */
class rcube_csv2vcard
{
    /**
     * CSV to vCard fields mapping
     *
     * @var array
     */
    protected $csv2vcard_map = array(
        // MS Outlook 2010
        'anniversary'           => 'anniversary',
        'assistants_name'       => 'assistant',
        'assistants_phone'      => 'phone:assistant',
        'birthday'              => 'birthday',
        'business_city'         => 'locality:work',
        'business_countryregion' => 'country:work',
        'business_fax'          => 'phone:work,fax',
        'business_phone'        => 'phone:work',
        'business_phone_2'      => 'phone:work2',
        'business_postal_code'  => 'zipcode:work',
        'business_state'        => 'region:work',
        'business_street'       => 'street:work',
        //'business_street_2'     => '',
        //'business_street_3'     => '',
        'car_phone'             => 'phone:car',
        'categories'            => 'categories',
        //'children'              => '',
        'company'               => 'organization',
        //'company_main_phone'    => '',
        'department'            => 'department',
        //'email_2_address'       => '', //@TODO
        //'email_2_type'          => '', //@TODO
        //'email_3_address'       => '', //@TODO
        //'email_3_type'          => '', //@TODO
        'email_address'         => 'email:main',
        //'email_type'            => '', //@TODO
        'first_name'            => 'firstname',
        'gender'                => 'gender',
        'home_city'             => 'locality:home',
        'home_countryregion'    => 'country:home',
        'home_fax'              => 'phone:home,fax',
        'home_phone'            => 'phone:home',
        'home_phone_2'          => 'phone:home2',
        'home_postal_code'      => 'zipcode:home',
        'home_state'            => 'region:home',
        'home_street'           => 'street:home',
        //'home_street_2'         => '',
        //'home_street_3'         => '',
        //'initials'              => '',
        //'isdn'                  => '',
        'job_title'             => 'jobtitle',
        //'keywords'              => '',
        //'language'              => '',
        'last_name'             => 'surname',
        //'location'              => '',
        'managers_name'         => 'manager',
        'middle_name'           => 'middlename',
        //'mileage'               => '',
        'mobile_phone'          => 'phone:cell',
        'notes'                 => 'notes',
        //'office_location'       => '',
        'other_city'            => 'locality:other',
        'other_countryregion'   => 'country:other',
        'other_fax'             => 'phone:other,fax',
        'other_phone'           => 'phone:other',
        'other_postal_code'     => 'zipcode:other',
        'other_state'           => 'region:other',
        'other_street'          => 'street:other',
        //'other_street_2'        => '',
        //'other_street_3'        => '',
        'pager'                 => 'phone:pager',
        'primary_phone'         => 'phone:pref',
        //'profession'            => '',
        //'radio_phone'           => '',
        'spouse'                => 'spouse',
        'suffix'                => 'suffix',
        'title'                 => 'title',
        'web_page'              => 'website:homepage',

        // Thunderbird
        'birth_day'             => 'birthday-d',
        'birth_month'           => 'birthday-m',
        'birth_year'            => 'birthday-y',
        'display_name'          => 'displayname',
        'fax_number'            => 'phone:fax',
        'home_address'          => 'street:home',
        //'home_address_2'        => '',
        'home_country'          => 'country:home',
        'home_zipcode'          => 'zipcode:home',
        'mobile_number'         => 'phone:cell',
        'nickname'              => 'nickname',
        'organization'          => 'organization',
        'pager_number'          => 'phone:pager',
        'primary_email'         => 'email:pref',
        'secondary_email'       => 'email:other',
        'web_page_1'            => 'website:homepage',
        'web_page_2'            => 'website:other',
        'work_phone'            => 'phone:work',
        'work_address'          => 'street:work',
        //'work_address_2'        => '',
        'work_country'          => 'country:work',
        'work_zipcode'          => 'zipcode:work',
    );

    /**
     * CSV label to text mapping for English
     *
     * @var array
     */
    protected $label_map = array(
        // MS Outlook 2010
        'anniversary'       => "Anniversary",
        'assistants_name'   => "Assistant's Name",
        'assistants_phone'  => "Assistant's Phone",
        'birthday'          => "Birthday",
        'business_city'     => "Business City",
        'business_countryregion' => "Business Country/Region",
        'business_fax'      => "Business Fax",
        'business_phone'    => "Business Phone",
        'business_phone_2'  => "Business Phone 2",
        'business_postal_code' => "Business Postal Code",
        'business_state'    => "Business State",
        'business_street'   => "Business Street",
        //'business_street_2' => "Business Street 2",
        //'business_street_3' => "Business Street 3",
        'car_phone'         => "Car Phone",
        'categories'        => "Categories",
        //'children'          => "Children",
        'company'           => "Company",
        //'company_main_phone' => "Company Main Phone",
        'department'        => "Department",
        //'directory_server'  => "Directory Server",
        //'email_2_address'   => "E-mail 2 Address", //@TODO
        //'email_2_type'      => "E-mail 2 Type", //@TODO
        //'email_3_address'   => "E-mail 3 Address", //@TODO
        //'email_3_type'      => "E-mail 3 Type", //@TODO
        'email_address'     => "E-mail Address",
        //'email_type'        => "E-mail Type", //@TODO
        'first_name'        => "First Name",
        'gender'            => "Gender",
        'home_city'         => "Home City",
        'home_countryregion' => "Home Country/Region",
        'home_fax'          => "Home Fax",
        'home_phone'        => "Home Phone",
        'home_phone_2'      => "Home Phone 2",
        'home_postal_code'  => "Home Postal Code",
        'home_state'        => "Home State",
        'home_street'       => "Home Street",
        //'home_street_2'     => "Home Street 2",
        //'home_street_3'     => "Home Street 3",
        //'initials'          => "Initials",
        //'isdn'              => "ISDN",
        'job_title'         => "Job Title",
        //'keywords'          => "Keywords",
        //'language'          => "Language",
        'last_name'         => "Last Name",
        //'location'          => "Location",
        'managers_name'     => "Manager's Name",
        'middle_name'       => "Middle Name",
        //'mileage'           => "Mileage",
        'mobile_phone'      => "Mobile Phone",
        'notes'             => "Notes",
        //'office_location'   => "Office Location",
        'other_city'        => "Other City",
        'other_countryregion' => "Other Country/Region",
        'other_fax'         => "Other Fax",
        'other_phone'       => "Other Phone",
        'other_postal_code' => "Other Postal Code",
        'other_state'       => "Other State",
        'other_street'      => "Other Street",
        //'other_street_2'    => "Other Street 2",
        //'other_street_3'    => "Other Street 3",
        'pager'             => "Pager",
        'primary_phone'     => "Primary Phone",
        //'profession'        => "Profession",
        //'radio_phone'       => "Radio Phone",
        'spouse'            => "Spouse",
        'suffix'            => "Suffix",
        'title'             => "Title",
        'web_page'          => "Web Page",

        // Thunderbird
        'birth_day'         => "Birth Day",
        'birth_month'       => "Birth Month",
        'birth_year'        => "Birth Year",
        'display_name'      => "Display Name",
        'fax_number'        => "Fax Number",
        'home_address'      => "Home Address",
        //'home_address_2'    => "Home Address 2",
        'home_country'      => "Home Country",
        'home_zipcode'      => "Home ZipCode",
        'mobile_number'     => "Mobile Number",
        'nickname'          => "Nickname",
        'organization'      => "Organization",
        'pager_number'      => "Pager Namber",
        'primary_email'     => "Primary Email",
        'secondary_email'   => "Secondary Email",
        'web_page_1'        => "Web Page 1",
        'web_page_2'        => "Web Page 2",
        'work_phone'        => "Work Phone",
        'work_address'      => "Work Address",
        //'work_address_2'    => "Work Address 2",
        'work_country'      => "Work Country",
        'work_zipcode'      => "Work ZipCode",
    );

    protected $vcards = array();
    protected $map    = array();


    /**
     * Class constructor
     *
     * @param string $lang File language
     */
    public function __construct($lang = 'en_US')
    {
        // Localize fields map
        if ($lang && $lang != 'en_US') {
            if (file_exists(INSTALL_PATH . "program/localization/$lang/csv2vcard.inc")) {
                include INSTALL_PATH . "program/localization/$lang/csv2vcard.inc";
            }

            if (!empty($map)) {
                $this->label_map = array_merge($this->label_map, $map);
            }
        }

        $this->label_map = array_flip($this->label_map);
    }

    /**
     *
     */
    public function import($csv)
    {
        // convert to UTF-8
        $head     = substr($csv, 0, 4096);
        $fallback = rcube::get_instance()->config->get('default_charset', 'ISO-8859-1'); // fallback to Latin-1?
        $charset  = rcube_charset::detect($head, RCMAIL_CHARSET);
        $csv      = rcube_charset::convert($csv, $charset);
        $head     = '';

        $this->map = array();

        // Parse file
        foreach (preg_split("/[\r\n]+/", $csv) as $i => $line) {
            $line = trim($line);
            if (empty($line)) {
                continue;
            }

            $elements = rcube_utils::explode_quoted_string(',', $line);

            if (empty($elements)) {
                continue;
            }

            // Parse header
            if (empty($this->map)) {
                $this->parse_header($elements);
                if (empty($this->map)) {
                    break;
                }
            }
            // Parse data row
            else {
                $this->csv_to_vcard($elements);
            }
        }
    }

    /**
     * @return array rcube_vcard List of vcards
     */
    public function export()
    {
        return $this->vcards;
    }

    /**
     * Parse CSV header line, detect fields mapping
     */
    protected function parse_header($elements)
     {
        for ($i = 0, $size = count($elements); $i<$size; $i++) {
            $label = $this->label_map[$elements[$i]];
            if ($label && !empty($this->csv2vcard_map[$label])) {
                $this->map[$i] = $this->csv2vcard_map[$label];
            }
        }
    }

    /**
     * Convert CSV data row to vCard
     */
    protected function csv_to_vcard($data)
    {
        $contact = array();
        foreach ($this->map as $idx => $name) {
            $value = $data[$idx];
            if ($value !== null && $value !== '') {
                $contact[$name] = $value;
            }
        }

        if (empty($contact)) {
            return;
        }

        // Handle special values
        if (!empty($contact['birthday-d']) && !empty($contact['birthday-m']) && !empty($contact['birthday-y'])) {
            $contact['birthday'] = $contact['birthday-y'] .'-' .$contact['birthday-m'] . '-' . $contact['birthday-d'];
        }

        // Create vcard object
        $vcard = new rcube_vcard();
        foreach ($contact as $name => $value) {
            $name = explode(':', $name);
            $vcard->set($name[0], $value, $name[1]);
        }

        // add to the list
        $this->vcards[] = $vcard;
    }
}

 program/localization/en_US/csv2vcard.inc

New file
@@ -0,0 +1,93 @@
<?php

/*
 +-----------------------------------------------------------------------+
 | language/en_US/csv2vcard.inc                                          |
 |                                                                       |
 | Language file of the Roundcube Webmail client                         |
 | Copyright (C) 2005-2012, The Roundcube Dev Team                       |
 |                                                                       |
 | Licensed under the GNU General Public License version 3 or            |
 | any later version with exceptions for skins & plugins.                |
 | See the README file for a full license statement.                     |
 |                                                                       |
 +-----------------------------------------------------------------------+
 | Author: Aleksander Machniak <alec@alec.pl>                            |
 +-----------------------------------------------------------------------+
*/

// This is a list of CSV column names specified in CSV file header
// These must be original texts used in Outlook/Thunderbird exported csv files
// Encoding UTF-8

$map = array();

// MS Outlook 2010
$map['anniversary'] = "Anniversary";
$map['assistants_name'] = "Assistant's Name";
$map['assistants_phone'] = "Assistant's Phone";
$map['birthday'] = "Birthday";
$map['business_city'] = "Business City";
$map['business_countryregion'] = "Business Country/Region";
$map['business_fax'] = "Business Fax";
$map['business_phone'] = "Business Phone";
$map['business_phone_2'] = "Business Phone 2";
$map['business_postal_code'] = "Business Postal Code";
$map['business_state'] = "Business State";
$map['business_street'] = "Business Street";
$map['car_phone'] = "Car Phone";
$map['categories'] = "Categories";
$map['company'] = "Company";
$map['department'] = "Department";
$map['email_address'] = "E-mail Address";
$map['first_name'] = "First Name";
$map['gender'] = "Gender";
$map['home_city'] = "Home City";
$map['home_countryregion'] = "Home Country/Region";
$map['home_fax'] = "Home Fax";
$map['home_phone'] = "Home Phone";
$map['home_phone_2'] = "Home Phone 2";
$map['home_postal_code'] = "Home Postal Code";
$map['home_state'] = "Home State";
$map['home_street'] = "Home Street";
$map['job_title'] = "Job Title";
$map['last_name'] = "Last Name";
$map['managers_name'] = "Manager's Name";
$map['middle_name'] = "Middle Name";
$map['mobile_phone'] = "Mobile Phone";
$map['notes'] = "Notes";
$map['other_city'] = "Other City";
$map['other_countryregion'] = "Other Country/Region";
$map['other_fax'] = "Other Fax";
$map['other_phone'] = "Other Phone";
$map['other_postal_code'] = "Other Postal Code";
$map['other_state'] = "Other State";
$map['other_street'] = "Other Street";
$map['pager'] = "Pager";
$map['primary_phone'] = "Primary Phone";
$map['spouse'] = "Spouse";
$map['suffix'] = "Suffix";
$map['title'] = "Title";
$map['web_page'] = "Web Page";

// Thunderbird
$map['birth_day'] = "Birth Day";
$map['birth_month'] = "Birth Month";
$map['birth_year'] = "Birth Year";
$map['display_name'] = "Display Name";
$map['fax_number'] = "Fax Number";
$map['home_address'] = "Home Address";
$map['home_country'] = "Home Country";
$map['home_zipcode'] = "Home ZipCode";
$map['mobile_number'] = "Mobile Number";
$map['nickname'] = "Nickname";
$map['organization'] = "Organization";
$map['pager_number'] = "Pager Namber";
$map['primary_email'] = "Primary Email";
$map['secondary_email'] = "Secondary Email";
$map['web_page_1'] = "Web Page 1";
$map['web_page_2'] = "Web Page 2";
$map['work_phone'] = "Work Phone";
$map['work_address'] = "Work Address";
$map['work_country'] = "Work Country";
$map['work_zipcode'] = "Work ZipCode";

 program/localization/en_US/labels.inc

@@ -354,7 +354,7 @@
$labels['importfromfile'] = 'Import from file:';
$labels['importtarget'] = 'Add new contacts to address book:';
$labels['importreplace'] = 'Replace the entire address book';
$labels['importtext'] = 'You can upload contacts from an existing address book.<br/>We currently support importing addresses from the <a href="http://en.wikipedia.org/wiki/VCard">vCard</a> data format.';
$labels['importdesc'] = 'You can upload contacts from an existing address book.<br/>We currently support importing addresses from the <a href="http://en.wikipedia.org/wiki/VCard">vCard</a> or CSV (comma-separated) data format.';
$labels['done'] = 'Done';

// settings

 program/localization/en_US/messages.inc

@@ -1,12 +1,11 @@
<?php

/*

 +-----------------------------------------------------------------------+
 | language/en_US/messages.inc                                           |
 |                                                                       |
 | Language file of the Roundcube Webmail client                         |
 | Copyright (C) 2005-2010, The Roundcube Dev Team                       |
 | Copyright (C) 2005-2012, The Roundcube Dev Team                       |
 |                                                                       |
 | Licensed under the GNU General Public License version 3 or            |
 | any later version with exceptions for skins & plugins.                |
@@ -15,9 +14,6 @@
 +-----------------------------------------------------------------------+
 | Author: Thomas Bruederli <roundcube@gmail.com>                        |
 +-----------------------------------------------------------------------+

 @version $Id$

*/

$messages = array();
@@ -125,7 +121,7 @@
$messages['contactremovedfromgroup'] = 'Successfully removed contacts from this group.';
$messages['nogroupassignmentschanged'] = 'No group assignments changed.';
$messages['importwait'] = 'Importing, please wait...';
$messages['importerror'] = 'Import failed! The uploaded file is not a valid vCard file.';
$messages['importformaterror'] = 'Import failed! The uploaded file is not a valid import data file.';
$messages['importconfirm'] = '<b>Successfully imported $inserted contacts</b>';
$messages['importconfirmskipped'] = '<b>Skipped $skipped existing entries</b>';
$messages['opnotpermitted'] = 'Operation not permitted!';

 program/steps/addressbook/import.inc

@@ -64,7 +64,7 @@
  $OUTPUT->add_label('selectimportfile','importwait');
  $OUTPUT->add_gui_object('importform', $attrib['id']);

  $out = html::p(null, Q(rcube_label('importtext'), 'show'));
  $out = html::p(null, Q(rcube_label('importdesc'), 'show'));

  $out .= $OUTPUT->form_tag(array(
      'action' => $RCMAIL->url('import'),
@@ -159,11 +159,22 @@
                $upload_error = $err;
            }
            else {
                $file_content = file_get_contents($filepath);

                // let rcube_vcard do the hard work :-)
                $vcard_o = new rcube_vcard();
                $vcard_o->extend_fieldmap($CONTACTS->vcard_map);
                $v_list = $vcard_o->import($file_content);

                $v_list = $vcard_o->import(file_get_contents($filepath));
                if (!empty($v_list)) {
                    $vcards = array_merge($vcards, $v_list);
                    continue;
                }

                // no vCards found, try CSV
                $csv = new rcube_csv2vcard($_SESSION['language']);
                $csv->import($file_content);
                $v_list = $csv->export();

                if (!empty($v_list)) {
                    $vcards = array_merge($vcards, $v_list);
@@ -181,7 +192,7 @@
            $OUTPUT->show_message('fileuploaderror', 'error');
        }
        else {
            $OUTPUT->show_message('importerror', 'error');
            $OUTPUT->show_message('importformaterror', 'error');
        }
    }
    else {

 tests/Framework/Csv2vcard.php

New file
@@ -0,0 +1,57 @@
<?php

/**
 * Test class to test rcube_csv2vcard class
 *
 * @package Tests
 */
class Framework_Csv2vcard extends PHPUnit_Framework_TestCase
{

    function test_import_generic()
    {
        $csv = new rcube_csv2vcard;

        // empty input
        $csv->import('');
        $this->assertSame(array(), $csv->export());
    }

    function test_import_tb_plain()
    {
        $csv_text = file_get_contents(TESTS_DIR . '/src/Csv2vcard/tb_plain.csv');
        $vcf_text = file_get_contents(TESTS_DIR . '/src/Csv2vcard/tb_plain.vcf');

        $csv = new rcube_csv2vcard;
        $csv->import($csv_text);
        $result = $csv->export();
        $vcard = $result[0]->export(false);

        $this->assertCount(1, $result);

        $vcf_text = trim(str_replace("\r\n", "\n", $vcf_text));
        $vcard    = trim(str_replace("\r\n", "\n", $vcard));
        $this->assertEquals($vcf_text, $vcard);
    }

    function test_import_email()
    {
        $csv_text = file_get_contents(TESTS_DIR . '/src/Csv2vcard/email.csv');
        $vcf_text = file_get_contents(TESTS_DIR . '/src/Csv2vcard/email.vcf');

        $csv = new rcube_csv2vcard;
        $csv->import($csv_text);
        $result = $csv->export();

        $this->assertCount(4, $result);

        $vcard = '';
        foreach ($result as $vcf) {
            $vcard .= $vcf->export(false) . "\n";
        }

        $vcf_text = trim(str_replace("\r\n", "\n", $vcf_text));
        $vcard    = trim(str_replace("\r\n", "\n", $vcard));
        $this->assertEquals($vcf_text, $vcard);
    }
}

 tests/phpunit.xml

@@ -8,6 +8,7 @@
            <file>Framework/Cache.php</file>
            <file>Framework/Charset.php</file>
            <file>Framework/ContentFilter.php</file>
            <file>Framework/Csv2vcard.php</file>
            <file>Framework/Html.php</file>
            <file>Framework/Imap.php</file>
            <file>Framework/ImapGeneric.php</file>

 tests/src/Csv2vcard/email.csv

New file
@@ -0,0 +1,5 @@
Primary Email
test1@domain.tld
test2@domain.tld
test3@domain.tld
test4@domain.tld

 tests/src/Csv2vcard/email.vcf

New file
@@ -0,0 +1,20 @@
BEGIN:VCARD
VERSION:3.0
FN:test1@domain.tld
EMAIL;TYPE=INTERNET;TYPE=PREF:test1@domain.tld
END:VCARD
BEGIN:VCARD
VERSION:3.0
FN:test2@domain.tld
EMAIL;TYPE=INTERNET;TYPE=PREF:test2@domain.tld
END:VCARD
BEGIN:VCARD
VERSION:3.0
FN:test3@domain.tld
EMAIL;TYPE=INTERNET;TYPE=PREF:test3@domain.tld
END:VCARD
BEGIN:VCARD
VERSION:3.0
FN:test4@domain.tld
EMAIL;TYPE=INTERNET;TYPE=PREF:test4@domain.tld
END:VCARD

 tests/src/Csv2vcard/tb_plain.csv

New file
@@ -0,0 +1,2 @@
First Name,Last Name,Display Name,Nickname,Primary Email,Secondary Email,Screen Name,Work Phone,Home Phone,Fax Number,Pager Number,Mobile Number,Home Address,Home Address 2,Home City,Home State,Home ZipCode,Home Country,Work Address,Work Address 2,Work City,Work State,Work ZipCode,Work Country,Job Title,Department,Organization,Web Page 1,Web Page 2,Birth Year,Birth Month,Birth Day,Custom 1,Custom 2,Custom 3,Custom 4,Notes,
Firstname,Lastname,Displayname,Nick,test@domain.tld,next@domain.tld,,phone work,phone home,fax,pager,mobile,Priv address,,City,region,xx-xxx,USA,Addr work,,city,region,33-333,Poland,title,department,Organization,http://page.com,http://webpage.tld,1970,11,15,,,,,,

 tests/src/Csv2vcard/tb_plain.vcf

New file
@@ -0,0 +1,18 @@
BEGIN:VCARD
VERSION:3.0
FN:Displayname
N:Lastname;Firstname;;;
NICKNAME:Nick
EMAIL;TYPE=INTERNET;TYPE=PREF:test@domain.tld
EMAIL;TYPE=INTERNET;TYPE=OTHER:next@domain.tld
TEL;TYPE=work:phone work
TEL;TYPE=home:phone home
TEL;TYPE=fax:fax
TEL;TYPE=cell:mobile
TITLE:title
X-DEPARTMENT:department
ORG:Organization
URL;TYPE=homepage:http://page.com
URL;TYPE=other:http://webpage.tld
BDAY;VALUE=date:1970-11-15
END:VCARD