1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
|
<?php
/**
* PHPTAL templating engine
*
* PHP Version 5
*
* @category HTML
* @package PHPTAL
* @author Kornel Lesiński <kornel@aardvarkmedia.co.uk>
* @license http://www.gnu.org/licenses/lgpl.html GNU Lesser General Public License
* @version SVN: $Id:$
* @link http://phptal.org/
*/
class PHPTAL_Tokenizer
{
private $regex, $names, $offset, $str;
private $current_token, $current_value;
function __construct($str, array $tokens)
{
$this->offset = 0;
$this->str = $str;
$this->end = strlen($str);
$this->regex = '/('.str_replace('/', '\/', implode(')|(', $tokens)).')|(.)/Ssi';
$this->names = array_keys($tokens);
$this->names[] = 'OTHER';
}
function eof()
{
return $this->offset >= $this->end;
}
function skipSpace()
{
while ($this->current_token === 'SPACE') $this->nextToken();
}
function nextToken()
{
if ($this->offset >= $this->end) {
$this->current_value = null;
return $this->current_token = 'EOF';
}
//if (!preg_match_all($this->regex, $this->str, $m, PREG_SET_ORDER, $this->offset)) throw new Exception("FAIL {$this->regex} at {$this->offset}");
if (!preg_match($this->regex, $this->str, $m, null, $this->offset)) throw new Exception("FAIL {$this->regex} didn't match '{$this->str}' at {$this->offset}");
$this->offset += strlen($m[0]); // in bytes
$this->current_value = $m[0];
$this->current_token = $this->names[count($m)-2]; // -1 for usual length/offset confusion, and minus one extra for $m[0]
return $this->current_token;
}
function token()
{
return $this->current_token;
}
function tokenValue()
{
return $this->current_value;
}
}
|
