From 6af3b6dfdbd2be9de3a3d670b89f694e4fcffd69 Mon Sep 17 00:00:00 2001
From: Sébastien Kergreis <seb.kerro@gmail.com>
Date: Tue, 31 May 2016 15:44:09 -0500
Subject: Fix filters with UTF-8 characters

---
 tests/units/Core/Filter/LexerTest.php | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

(limited to 'tests/units')

diff --git a/tests/units/Core/Filter/LexerTest.php b/tests/units/Core/Filter/LexerTest.php
index d405e9df..c72231c4 100644
--- a/tests/units/Core/Filter/LexerTest.php
+++ b/tests/units/Core/Filter/LexerTest.php
@@ -178,4 +178,28 @@ class LexerTest extends Base
 
         $this->assertSame($expected, $lexer->tokenize('date:<=2016-01-01 something else'));
     }
+
+    public function testTokenizeWithUtf8Letters()
+    {
+        $lexer = new Lexer();
+        $lexer->setDefaultToken('myDefaultToken');
+
+        $expected = array(
+            'myDefaultToken' => array('àa éçùe'),
+        );
+
+        $this->assertSame($expected, $lexer->tokenize('àa éçùe'));
+    }
+
+    public function testTokenizeWithUtf8Numbers()
+    {
+        $lexer = new Lexer();
+        $lexer->setDefaultToken('myDefaultToken');
+
+        $expected = array(
+            'myDefaultToken' => array('६Δↈ五一'),
+        );
+
+        $this->assertSame($expected, $lexer->tokenize('६Δↈ五一'));
+    }
 }
-- 
cgit v1.2.3