From e515f37435db6cf883215f13f02391d8b2107d47 Mon Sep 17 00:00:00 2001 From: Frederic Guillot Date: Mon, 12 Oct 2015 14:44:28 -0400 Subject: Add user CSV import --- ChangeLog | 5 + app/Console/ProjectDailyColumnStatsExport.php | 4 +- app/Console/SubtaskExport.php | 4 +- app/Console/TaskExport.php | 4 +- app/Console/TransitionExport.php | 4 +- app/Controller/UserImport.php | 67 ++++++++ app/Core/Csv.php | 212 ++++++++++++++++++++++++++ app/Core/Request.php | 12 ++ app/Core/Response.php | 3 +- app/Core/Router.php | 3 +- app/Core/Tool.php | 21 --- app/Helper/Form.php | 17 +++ app/Model/Acl.php | 2 + app/Model/UserImport.php | 110 +++++++++++++ app/ServiceProvider/ClassProvider.php | 1 + app/Template/user/index.php | 3 +- app/Template/user_import/step1.php | 46 ++++++ tests/units/Core/CsvTest.php | 22 +++ tests/units/Core/RouterTest.php | 6 +- tests/units/Model/AclTest.php | 3 + 20 files changed, 514 insertions(+), 35 deletions(-) create mode 100644 app/Controller/UserImport.php create mode 100644 app/Core/Csv.php create mode 100644 app/Model/UserImport.php create mode 100644 app/Template/user_import/step1.php create mode 100644 tests/units/Core/CsvTest.php diff --git a/ChangeLog b/ChangeLog index 9f09bac5..504cd984 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,9 +1,14 @@ Version 1.0.20 (unreleased) --------------------------- +New features: + +* Add users CSV import + Improvements: * Allow to change comments sorting +* Add the possibility to append or not custom filters Version 1.0.19 -------------- diff --git a/app/Console/ProjectDailyColumnStatsExport.php b/app/Console/ProjectDailyColumnStatsExport.php index b9830662..4db33b6a 100644 --- a/app/Console/ProjectDailyColumnStatsExport.php +++ b/app/Console/ProjectDailyColumnStatsExport.php @@ -2,7 +2,7 @@ namespace Console; -use Core\Tool; +use Core\Csv; use Symfony\Component\Console\Input\InputArgument; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Output\OutputInterface; @@ -28,7 +28,7 @@ class ProjectDailyColumnStatsExport extends Base ); if (is_array($data)) { - Tool::csv($data); + Csv::output($data); } } } diff --git a/app/Console/SubtaskExport.php b/app/Console/SubtaskExport.php index 167a9225..9816574b 100644 --- a/app/Console/SubtaskExport.php +++ b/app/Console/SubtaskExport.php @@ -2,7 +2,7 @@ namespace Console; -use Core\Tool; +use Core\Csv; use Symfony\Component\Console\Input\InputArgument; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Output\OutputInterface; @@ -28,7 +28,7 @@ class SubtaskExport extends Base ); if (is_array($data)) { - Tool::csv($data); + Csv::output($data); } } } diff --git a/app/Console/TaskExport.php b/app/Console/TaskExport.php index 2ecd45e5..862c1b0d 100644 --- a/app/Console/TaskExport.php +++ b/app/Console/TaskExport.php @@ -2,7 +2,7 @@ namespace Console; -use Core\Tool; +use Core\Csv; use Symfony\Component\Console\Input\InputArgument; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Output\OutputInterface; @@ -28,7 +28,7 @@ class TaskExport extends Base ); if (is_array($data)) { - Tool::csv($data); + Csv::output($data); } } } diff --git a/app/Console/TransitionExport.php b/app/Console/TransitionExport.php index ad988c54..27d8de64 100644 --- a/app/Console/TransitionExport.php +++ b/app/Console/TransitionExport.php @@ -2,7 +2,7 @@ namespace Console; -use Core\Tool; +use Core\Csv; use Symfony\Component\Console\Input\InputArgument; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Output\OutputInterface; @@ -28,7 +28,7 @@ class TransitionExport extends Base ); if (is_array($data)) { - Tool::csv($data); + Csv::output($data); } } } diff --git a/app/Controller/UserImport.php b/app/Controller/UserImport.php new file mode 100644 index 00000000..e31ddbbb --- /dev/null +++ b/app/Controller/UserImport.php @@ -0,0 +1,67 @@ +response->html($this->template->layout('user_import/step1', array( + 'values' => $values, + 'errors' => $errors, + 'max_size' => ini_get('upload_max_filesize'), + 'delimiters' => Csv::getDelimiters(), + 'enclosures' => Csv::getEnclosures(), + 'title' => t('Import users from CSV file'), + ))); + } + + /** + * Process CSV file + * + */ + public function step2() + { + $values = $this->request->getValues(); + $filename = $this->request->getFilePath('file'); + + if (! file_exists($filename)) { + $this->step1($values, array('file' => array(t('Unable to read your file')))); + } + + $csv = new Csv($values['delimiter'], $values['enclosure']); + $csv->setColumnMapping($this->userImport->getColumnMapping()); + $csv->read($filename, array($this->userImport, 'import')); + + if ($this->userImport->counter > 0) { + $this->session->flash(t('%d user(s) have been imported successfully.', $this->userImport->counter)); + } + else { + $this->session->flash(t('Nothing have been imported!')); + } + + $this->response->redirect($this->helper->url->to('userImport', 'step1')); + } + + /** + * Generate template + * + */ + public function template() + { + $this->response->forceDownload('users.csv'); + $this->response->csv(array($this->userImport->getColumnMapping())); + } +} diff --git a/app/Core/Csv.php b/app/Core/Csv.php new file mode 100644 index 00000000..6e7816f6 --- /dev/null +++ b/app/Core/Csv.php @@ -0,0 +1,212 @@ +delimiter = $delimiter; + $this->enclosure = $enclosure; + } + + /** + * Get list of delimiters + * + * @static + * @access public + * @return array + */ + public static function getDelimiters() + { + return array( + ',' => t('Comma'), + ';' => t('Semi-colon'), + '\t' => t('Tab'), + '|' => t('Vertical bar'), + ); + } + + /** + * Get list of enclosures + * + * @static + * @access public + * @return array + */ + public static function getEnclosures() + { + return array( + '"' => t('Double Quote'), + "'" => t('Single Quote'), + '' => t('None'), + ); + } + + /** + * Check boolean field value + * + * @static + * @access public + * @return integer + */ + public static function getBooleanValue($value) + { + if (! empty($value)) { + $value = trim(strtolower($value)); + return $value === '1' || $value{0} === 't' ? 1 : 0; + } + + return 0; + } + + /** + * Output CSV file to standard output + * + * @static + * @access public + * @param array $rows + */ + public static function output(array $rows) + { + $csv = new static; + $csv->write('php://output', $rows); + } + + /** + * Define column mapping between CSV and SQL columns + * + * @access public + * @param array $columns + * @return Csv + */ + public function setColumnMapping(array $columns) + { + $this->columns = $columns; + return $this; + } + + /** + * Read CSV file + * + * @access public + * @param string $filename + * @param \Closure $callback Example: function(array $row, $line_number) + * @return Csv + */ + public function read($filename, $callback) + { + $file = new SplFileObject($filename); + $file->setFlags(SplFileObject::READ_CSV); + $file->setCsvControl($this->delimiter, $this->enclosure); + $line_number = 0; + + foreach ($file as $row) { + $row = $this->filterRow($row); + + if (! empty($row) && $line_number > 0) { + call_user_func_array($callback, array($this->associateColumns($row), $line_number)); + } + + $line_number++; + } + + return $this; + } + + /** + * Write CSV file + * + * @access public + * @param string $filename + * @param array $rows + * @return Csv + */ + public function write($filename, array $rows) + { + $file = new SplFileObject($filename, 'w'); + + foreach ($rows as $row) { + $file->fputcsv($row, $this->delimiter, $this->enclosure); + } + + return $this; + } + + /** + * Associate columns header with row values + * + * @access private + * @param array $row + * @return array + */ + private function associateColumns(array $row) + { + $line = array(); + $index = 0; + + foreach ($this->columns as $sql_name => $csv_name) { + if (isset($row[$index])) { + $line[$sql_name] = $row[$index]; + } + else { + $line[$sql_name] = ''; + } + + $index++; + } + + return $line; + } + + /** + * Filter empty rows + * + * @access private + * @param array $row + * @return array + */ + private function filterRow(array $row) + { + return array_filter($row); + } +} diff --git a/app/Core/Request.php b/app/Core/Request.php index 1eff66fa..d0fcdb8e 100644 --- a/app/Core/Request.php +++ b/app/Core/Request.php @@ -102,6 +102,18 @@ class Request return ''; } + /** + * Get the path of an uploaded file + * + * @access public + * @param string $name Form file name + * @return string + */ + public function getFilePath($name) + { + return isset($_FILES[$name]['tmp_name']) ? $_FILES[$name]['tmp_name'] : ''; + } + /** * Return true if the HTTP request is sent with the POST method * diff --git a/app/Core/Response.php b/app/Core/Response.php index f8ca015c..71d99524 100644 --- a/app/Core/Response.php +++ b/app/Core/Response.php @@ -87,8 +87,9 @@ class Response { $this->status($status_code); $this->nocache(); + header('Content-Type: text/csv'); - Tool::csv($data); + Csv::output($data); exit; } diff --git a/app/Core/Router.php b/app/Core/Router.php index 93d266bb..55ebe4a8 100644 --- a/app/Core/Router.php +++ b/app/Core/Router.php @@ -197,7 +197,7 @@ class Router extends Base */ public function sanitize($value, $default_value) { - return ! ctype_alpha($value) || empty($value) ? $default_value : strtolower($value); + return ! preg_match('/^[a-zA-Z_0-9]+$/', $value) ? $default_value : $value; } /** @@ -218,6 +218,7 @@ class Router extends Base list($this->controller, $this->action) = $this->findRoute($this->getPath($uri, $query_string)); // TODO: add plugin for routes $plugin = ''; } + $class = empty($plugin) ? '\Controller\\'.ucfirst($this->controller) : '\Plugin\\'.ucfirst($plugin).'\Controller\\'.ucfirst($this->controller); $instance = new $class($this->container); diff --git a/app/Core/Tool.php b/app/Core/Tool.php index 887c8fb3..39e42b83 100644 --- a/app/Core/Tool.php +++ b/app/Core/Tool.php @@ -12,27 +12,6 @@ use Pimple\Container; */ class Tool { - /** - * Write a CSV file - * - * @static - * @access public - * @param array $rows Array of rows - * @param string $filename Output filename - */ - public static function csv(array $rows, $filename = 'php://output') - { - $fp = fopen($filename, 'w'); - - if (is_resource($fp)) { - foreach ($rows as $fields) { - fputcsv($fp, $fields); - } - - fclose($fp); - } - } - /** * Get the mailbox hash from an email address * diff --git a/app/Helper/Form.php b/app/Helper/Form.php index a37cc81a..e8664771 100644 --- a/app/Helper/Form.php +++ b/app/Helper/Form.php @@ -177,6 +177,23 @@ class Form extends \Core\Base return $html; } + /** + * Display file field + * + * @access public + * @param string $name + * @param array $errors + * @param boolean $multiple + * @return string + */ + public function file($name, array $errors = array(), $multiple = false) + { + $html = ''; + $html .= $this->errorList($errors, $name); + + return $html; + } + /** * Display a input field * diff --git a/app/Model/Acl.php b/app/Model/Acl.php index 675ca36e..d05e4f77 100644 --- a/app/Model/Acl.php +++ b/app/Model/Acl.php @@ -88,6 +88,7 @@ class Acl extends Base */ private $admin_acl = array( 'user' => array('index', 'create', 'save', 'remove', 'authentication'), + 'userimport' => '*', 'config' => '*', 'link' => '*', 'currency' => '*', @@ -117,6 +118,7 @@ class Acl extends Base */ public function matchAcl(array $acl, $controller, $action) { + $controller = strtolower($controller); $action = strtolower($action); return isset($acl[$controller]) && $this->hasAction($action, $acl[$controller]); } diff --git a/app/Model/UserImport.php b/app/Model/UserImport.php new file mode 100644 index 00000000..afae0a48 --- /dev/null +++ b/app/Model/UserImport.php @@ -0,0 +1,110 @@ + 'Username', + 'password' => 'Password', + 'email' => 'Email', + 'name' => 'Full Name', + 'is_admin' => 'Administrator', + 'is_project_admin' => 'Project Administrator', + 'is_ldap_user' => 'Remote User', + ); + } + + /** + * Import a single row + * + * @access public + * @param array $row + * @param integer $line_number + */ + public function import(array $row, $line_number) + { + $row = $this->prepare($row); + + if ($this->validateCreation($row)) { + if ($this->user->create($row)) { + $this->logger->debug('UserImport: imported successfully line '.$line_number); + $this->counter++; + } + else { + $this->logger->error('UserImport: creation error at line '.$line_number); + } + } + else { + $this->logger->error('UserImport: validation error at line '.$line_number); + } + } + + /** + * Format row before validation + * + * @access public + * @param array $data + * @return array + */ + public function prepare(array $row) + { + $row['username'] = strtolower($row['username']); + + foreach (array('is_admin', 'is_project_admin', 'is_ldap_user') as $field) { + $row[$field] = csv::getBooleanValue($row[$field]); + } + + $this->removeEmptyFields($row, array('password', 'email', 'name')); + + return $row; + } + + /** + * Validate user creation + * + * @access public + * @param array $values + * @return boolean + */ + public function validateCreation(array $values) + { + $v = new Validator($values, array( + new Validators\MaxLength('username', t('The maximum length is %d characters', 50), 50), + new Validators\Unique('username', t('The username must be unique'), $this->db->getConnection(), User::TABLE, 'id'), + new Validators\MinLength('password', t('The minimum length is %d characters', 6), 6), + new Validators\Email('email', t('Email address invalid')), + new Validators\Integer('is_admin', t('This value must be an integer')), + new Validators\Integer('is_project_admin', t('This value must be an integer')), + new Validators\Integer('is_ldap_user', t('This value must be an integer')), + )); + + return $v->execute(); + } +} diff --git a/app/ServiceProvider/ClassProvider.php b/app/ServiceProvider/ClassProvider.php index 5d157749..ac8fa750 100644 --- a/app/ServiceProvider/ClassProvider.php +++ b/app/ServiceProvider/ClassProvider.php @@ -65,6 +65,7 @@ class ClassProvider implements ServiceProviderInterface 'TaskValidator', 'Transition', 'User', + 'UserImport', 'UserSession', 'Webhook', ), diff --git a/app/Template/user/index.php b/app/Template/user/index.php index d74aa748..4008b920 100644 --- a/app/Template/user/index.php +++ b/app/Template/user/index.php @@ -4,10 +4,10 @@ -
isEmpty()): ?>

@@ -62,5 +62,4 @@ -
diff --git a/app/Template/user_import/step1.php b/app/Template/user_import/step1.php new file mode 100644 index 00000000..7256bfa6 --- /dev/null +++ b/app/Template/user_import/step1.php @@ -0,0 +1,46 @@ +
+
+ user->isAdmin()): ?> +
    +
  • url->link(t('All users'), 'user', 'index') ?>
    • +
  • url->link(t('New local user'), 'user', 'create') ?>
    • +
  • url->link(t('New remote user'), 'user', 'create', array('remote' => 1)) ?>
    • +
+ +
+
+

+
+
+ form->csrf() ?> + + form->label(t('Delimiter'), 'delimiter') ?> + form->select('delimiter', $delimiters, $values) ?> + + form->label(t('Enclosure'), 'enclosure') ?> + form->select('enclosure', $enclosures, $values) ?> + + form->label(t('CSV File'), 'file') ?> + form->file('file', $errors) ?> + +

text->bytes($max_size) : $max_size ?>

+ +
+ +
+
+
+

+
+
+
    +
    • +
    • +
    • +
    • +
    • +
    • +
+
+

url->link(t('Download CSV template'), 'userImport', 'template') ?>

+
\ No newline at end of file diff --git a/tests/units/Core/CsvTest.php b/tests/units/Core/CsvTest.php new file mode 100644 index 00000000..1534584e --- /dev/null +++ b/tests/units/Core/CsvTest.php @@ -0,0 +1,22 @@ +assertEquals(1, Csv::getBooleanValue('1')); + $this->assertEquals(1, Csv::getBooleanValue('True')); + $this->assertEquals(1, Csv::getBooleanValue('t')); + $this->assertEquals(1, Csv::getBooleanValue('TRUE')); + $this->assertEquals(1, Csv::getBooleanValue('true')); + $this->assertEquals(1, Csv::getBooleanValue('T')); + + $this->assertEquals(0, Csv::getBooleanValue('0')); + $this->assertEquals(0, Csv::getBooleanValue('123')); + $this->assertEquals(0, Csv::getBooleanValue('anything')); + } +} diff --git a/tests/units/Core/RouterTest.php b/tests/units/Core/RouterTest.php index 99c49ba8..56a87662 100644 --- a/tests/units/Core/RouterTest.php +++ b/tests/units/Core/RouterTest.php @@ -10,11 +10,13 @@ class RouterTest extends Base { $r = new Router($this->container); - $this->assertEquals('plop', $r->sanitize('PloP', 'default')); + $this->assertEquals('PloP', $r->sanitize('PloP', 'default')); $this->assertEquals('default', $r->sanitize('', 'default')); $this->assertEquals('default', $r->sanitize('123-AB', 'default')); $this->assertEquals('default', $r->sanitize('R&D', 'default')); - $this->assertEquals('default', $r->sanitize('Test123', 'default')); + $this->assertEquals('Test123', $r->sanitize('Test123', 'default')); + $this->assertEquals('Test_123', $r->sanitize('Test_123', 'default')); + $this->assertEquals('userImport', $r->sanitize('userImport', 'default')); } public function testPath() diff --git a/tests/units/Model/AclTest.php b/tests/units/Model/AclTest.php index 3cb28a77..205e7ee3 100644 --- a/tests/units/Model/AclTest.php +++ b/tests/units/Model/AclTest.php @@ -17,6 +17,7 @@ class AclTest extends Base 'controller3' => '*', 'controller5' => '-', 'controller6' => array(), + 'controllera' => '*', ); $acl = new Acl($this->container); @@ -30,6 +31,8 @@ class AclTest extends Base $this->assertFalse($acl->matchAcl($acl_rules, 'controller4', 'anything')); $this->assertFalse($acl->matchAcl($acl_rules, 'controller5', 'anything')); $this->assertFalse($acl->matchAcl($acl_rules, 'controller6', 'anything')); + $this->assertTrue($acl->matchAcl($acl_rules, 'ControllerA', 'anything')); + $this->assertTrue($acl->matchAcl($acl_rules, 'controllera', 'anything')); } public function testPublicActions() -- cgit v1.2.3