metagen/parser.inc.php

978 lines
26 KiB
PHP

<?php
class mtgMetaInfoParser
{
private $config = array();
private $current_meta;
private $parsed_files = array();
private $file_stack = array();
private $module = null;
private $file = "";
private $source = "";
private $cursor = 0;
private $line = 0;
private $token = "";
private $attribute = "";
private $idltypes = array();
private $token_strs = array();
private $shared_tokens = array();
const T_EOF = 1001;
const T_StringConstant = 1002;
const T_IntegerConstant = 1003;
const T_FloatConstant = 1004;
const T_Enum = 1005;
const T_RPC = 1006;
const T_End = 1007;
const T_Identifier = 1008;
const T_Struct = 1009;
const T_Prop = 1010;
const T_Extends = 1011;
const T_Func = 1012;
const T_RawStringConstant = 1013;
const T_Interface = 1014;
const T_Implements = 1015;
const T_string = 1020;
const T_uint32 = 1021;
const T_int32 = 1022;
const T_uint16 = 1023;
const T_int16 = 1024;
const T_uint8 = 1025;
const T_int8 = 1026;
const T_float = 1027;
const T_uint64 = 1028;
const T_int64 = 1029;
const T_bool = 1030;
const T_blob = 1031;
function __construct($config = array())
{
self::addDefaultTokens($config);
$this->config = $config;
if(!isset($this->config['include_path']))
$this->config['include_path'] = array('.');
$this->idltypes = array(
"string" => self::T_string,
"uint32" => self::T_uint32,
"int32" => self::T_int32,
"uint16" => self::T_uint16,
"int16" => self::T_int16,
"uint8" => self::T_uint8,
"int8" => self::T_int8,
"float" => self::T_float,
"double" => self::T_float,
"uint64" => self::T_uint64,
"int64" => self::T_int64,
"bool" => self::T_bool,
"blob" => self::T_blob,
);
$this->token_strs = array_flip($this->idltypes);
$this->token_strs[self::T_EOF] = '<EOF>';
$this->token_strs[self::T_StringConstant] = '<StringConstant>';
$this->token_strs[self::T_RawStringConstant] = '<RawStringConstant>';
$this->token_strs[self::T_IntegerConstant] = '<IntegerConstant>';
$this->token_strs[self::T_FloatConstant] = '<FloatConstant>';
$this->token_strs[self::T_Enum] = '<enum>';
$this->token_strs[self::T_RPC] = '<RPC>';
$this->token_strs[self::T_End] = '<end>';
$this->token_strs[self::T_Identifier] = '<Identifier>';
$this->token_strs[self::T_Struct] = '<struct>';
$this->token_strs[self::T_Interface] = '<interface>';
$this->token_strs[self::T_Prop] = '<@prop>';
$this->token_strs[self::T_Extends] = '<extends>';
$this->token_strs[self::T_Implements] = '<implements>';
$this->token_strs[self::T_Func] = '<func>';
}
function addDefaultTokens(array &$config)
{
if(!isset($config['valid_tokens']))
$config['valid_tokens'] = array();
$config['valid_tokens'][] = 'shared_tokens';
$config['valid_tokens'][] = 'enum_override';
}
function parse(mtgMetaInfo $meta, $raw_file)
{
$this->current_meta = $meta;
$file = realpath($raw_file);
if($file === false)
throw new Exception("No such file '$raw_file'");
$this->_parse($file);
mtgTypeRef::checkAllResolved();
}
private function _parse($file)
{
if(isset($this->parsed_files[$file]))
return;
$module = new mtgMetaParsedModule($file);
$this->parsed_files[$file] = $module;
$this->file_stack[] = $file;
$source = file_get_contents($file);
$is_php = false;
try
{
if($source === false)
throw new Exception("Could not read file '$file'");
//PHP include
if(strpos($source, '<?php') === 0)
{
include_once($file);
$is_php = true;
}
else
$this->_resolveIncludes($module, $source);
}
catch(Exception $e)
{
throw new Exception(end($this->file_stack) . " : " . $e->getMessage());
}
array_pop($this->file_stack);
if($is_php)
return;
$this->module = $module;
$this->file = $file;
$this->source = $source;
$this->cursor = 0;
$this->line = 1;
$this->shared_tokens = array();
try
{
$this->_next();
while($this->token != self::T_EOF)
{
//echo "TOKEN : " . $this->token . " " . $this->attribute . " " . $this->line . "\n";
if($this->token == self::T_Prop)
$this->_parseSharedTokens($this->_parsePropTokens());
else if($this->token == self::T_Enum)
$this->_parseEnum();
else if($this->token == self::T_Struct)
$this->_parseStruct();
else if($this->token == self::T_Interface)
$this->_parseInterface();
else if($this->token == self::T_Func)
$this->_parseFreeFunc();
else if($this->token == self::T_RPC)
$this->_parseRPC();
else
$this->_error("unexpected token ('" . $this->_toStr($this->token) . "' " . $this->attribute . ")");
}
}
catch(Exception $e)
{
throw new Exception("$file@{$this->line} : " . $e->getMessage() . " " . $e->getTraceAsString());
}
}
private function _parseInclude(mtgMetaParsedModule $module, $file)
{
$this->_parse($file);
$module->addInclude($this->parsed_files[$file]);
}
private function _parseSharedTokens(array $tokens)
{
if(!isset($tokens['shared_tokens']))
return;
$this->shared_tokens = json_decode($tokens['shared_tokens'], true);
if(!is_array($this->shared_tokens))
$this->_error("Invalid 'shared_tokens' formant, invalid JSON");
}
private function _parseType($can_be_multi = false)
{
$types = array();
while(true)
{
$type = null;
if($this->token == self::T_Func)
{
$func_type = $this->_parseFuncType();
$type = new mtgTypeRef($func_type, $this->module, $this->file, $this->line);
}
else if($this->token == self::T_Identifier)
{
$type_name = $this->_parseDotName();
$type = new mtgTypeRef($type_name, $this->module, $this->file, $this->line);
}
else
{
$type_name = $this->attribute;
$type = new mtgTypeRef(new mtgBuiltinType($type_name), $this->module, $this->file, $this->line);
$this->_next();
}
if($this->token == ord('['))
{
$this->_next();
$this->_checkThenNext(']');
$type = new mtgTypeRef(new mtgArrType($type), $this->module, $this->file, $this->line);
}
$types[] = $type;
if(!$can_be_multi)
break;
if($this->token != ord(','))
break;
$this->_next();
}
if(sizeof($types) > 1)
return new mtgTypeRef(new mtgMultiType($types), $this->module, $this->file, $this->line);
else
return $types[0];
}
private function _parseFuncType()
{
$ftype = new mtgMetaFunc('');
$this->_next();
$this->_checkThenNext('(');
$c = 0;
while(true)
{
if($this->token == ord(')'))
{
$this->_next();
break;
}
else if($c > 0)
{
$this->_checkThenNext(',');
}
$arg_type = $this->_parseType();
$c++;
$arg = new mtgMetaField("_$c", $arg_type);
$ftype->addArg($arg);
}
if($this->token == ord(':'))
{
$this->_next();
$ret_type = $this->_parseType(true/*can be multi-type*/);
$ftype->setReturnType($ret_type);
}
return $ftype;
}
private function _resolveIncludes(mtgMetaParsedModule $module, &$text)
{
$include_paths = $this->config['include_path'];
$result = array();
$lines = explode("\n", $text);
foreach($lines as $line)
{
if(preg_match('~^#include\s+(\S+)~', $line, $m))
{
$this->_processInclude($module, $m[1], $include_paths);
$result[] = "";
}
else
$result[] = $line;
}
$text = implode("\n", $result);
}
private function _processInclude(mtgMetaParsedModule $module, $include, array $include_paths)
{
$file = false;
foreach($include_paths as $include_path)
{
$file = realpath($include_path . "/" . $include);
if($file !== false)
break;
}
if($file === false)
throw new Exception("#include {$include} can't be resolved(include path is '". implode(':', $include_paths) . "')");
$this->_parseInclude($module, $file);
}
private function _parseEnumOrValues()
{
$values = array();
while(true)
{
if($this->token == self::T_Identifier)
{
$values[] = $this->attribute;
$this->_next();
if($this->token != ord('|'))
break;
else
$this->_next();
}
else
break;
}
return $values;
}
private function _parseEnum()
{
$this->_next();
$name = $this->_parseDotName();
$enum = new mtgMetaEnum($name);
$tokens = $this->shared_tokens;
if($this->token == self::T_Prop)
$tokens = array_merge($tokens, $this->_parsePropTokens());
$enum->setTokens($tokens);
$or_values = array();
while(true)
{
if($this->_nextIf(self::T_End))
break;
$key = $this->_checkThenNext(self::T_Identifier);
$this->_checkThenNext('=');
if($this->token == self::T_Identifier)
{
$or_values[$key] = $this->_parseEnumOrValues();
}
else
{
$value = $this->_checkThenNext(self::T_IntegerConstant);
$enum->addValue($key, $value);
}
}
$enum->addOrValues($or_values);
//NOTE: special case for enums when we allow to 'override' the original one,
// with additional values
if($enum->hasToken('enum_override'))
{
$existing = $this->current_meta->findUnit($enum->getId());
if(!$existing)
throw new Exception("Not found '{$name}' enum to override");
if(!($existing->object instanceof mtgMetaEnum))
throw new Exception("Not an enum struct '{$name}'");
$existing->object->override($enum);
}
else
$this->_addUnit(new mtgMetaInfoUnit($this->file, $enum));
}
private function _parseFields($next_doer)
{
$flds = array();
while(true)
{
if($next_doer())
break;
if($this->token == self::T_Identifier)
{
$name = $this->attribute;
$this->_next();
$this->_checkThenNext(':');
if($this->token == self::T_Identifier ||
$this->token == self::T_Func ||
($this->token >= self::T_string && $this->token <= self::T_blob))
{
$type = $this->_parseType();
$fld = new mtgMetaField($name, $type);
if($this->token == self::T_Prop)
$fld->setTokens($this->_parsePropTokens());
$flds[] = $fld;
}
else
$this->_error("type expected");
}
else
$this->_error("unexpected fields token");
}
return $flds;
}
private function _parseFuncs()
{
$end_token = self::T_End;
$funcs = array();
while(true)
{
$fn = $this->_parseFunc();
$funcs[] = $fn;
if($this->token == $end_token)
{
$this->_next();
break;
}
$this->_next();
}
return $funcs;
}
private function _parseDotName()
{
$dot_name = '';
while(true)
{
if($this->token != self::T_Identifier)
$this->_error("unexpected name token");
$dot_name .= $this->attribute;
$this->_next();
if($this->token != ord('.'))
break;
$dot_name .= '.';
$this->_next();
}
return $dot_name;
}
private function _parseFunc()
{
$name = $this->_parseDotName();
$fn = new mtgMetaFunc($name);
$this->_checkThenNext('(');
if($this->token == self::T_Prop)
$fn->setTokens($this->_parsePropTokens());
$args = $this->_parseFields(function()
{ return $this->_nextIf(')'); }
);
$fn->setArgs($args);
$ret_type = null;
if($this->token == ord(':'))
{
$this->_next();
if($this->token == self::T_Identifier ||
$this->token == self::T_Func ||
($this->token >= self::T_string && $this->token <= self::T_bool))
{
$ret_type = $this->_parseType(true/*can be multi-type*/);
$fn->setReturnType($ret_type);
}
else
$this->_error("unexpected func type token");
}
return $fn;
}
private function _addUnit(mtgMetaInfoUnit $unit)
{
$this->current_meta->addUnit($unit);
$this->module->addUnit($unit);
}
private function _parseFreeFunc()
{
$this->_next();
$fn = $this->_parseFunc();
$fn->setTokens(array_merge($this->shared_tokens, $fn->getTokens()));
$this->_addUnit(new mtgMetaInfoUnit($this->file, $fn));
}
private function _parseStruct()
{
$this->_next();
$name = $this->_parseDotName();
$parent = null;
if($this->token == self::T_Extends)
{
$this->_next();
$parent_name = $this->_checkThenNext(self::T_Identifier);
$parent = new mtgTypeRef($parent_name, $this->module, $this->file, $this->line);
}
$implements = array();
if($this->token == self::T_Implements)
{
do
{
$this->_next();
$if_name = $this->_checkThenNext(self::T_Identifier);
$implements[] = new mtgTypeRef($if_name, $this->module, $this->file, $this->line);
} while($this->token == ord(','));
}
$s = new mtgMetaStruct($name, array(), $parent, array(), $implements);
$this->_addUnit(new mtgMetaInfoUnit($this->file, $s));
$tokens = $this->shared_tokens;
if($this->token == self::T_Prop)
$tokens = array_merge($tokens, $this->_parsePropTokens());
$s->setTokens($tokens);
$seen_funcs = false;
$flds = $this->_parseFields(
function() use(&$seen_funcs)
{
if($this->_nextIf(self::T_End))
return true;
if($this->_nextIf(self::T_Func))
{
$seen_funcs = true;
return true;
}
}
);
foreach($flds as $fld)
$s->addField($fld);
if($seen_funcs)
{
$funcs = $this->_parseFuncs();
foreach($funcs as $fn)
$s->addFunc($fn);
}
}
private function _parseInterface()
{
$this->_next();
$name = $this->_parseDotName();
$s = new mtgMetaInterface($name);
$this->_addUnit(new mtgMetaInfoUnit($this->file, $s));
$tokens = $this->shared_tokens;
if($this->token == self::T_Prop)
$tokens = array_merge($tokens, $this->_parsePropTokens());
$s->setTokens($tokens);
$this->_next();
$funcs = $this->_parseFuncs();
foreach($funcs as $fn)
$s->addFunc($fn);
}
private function _parseRPC()
{
$this->_next();
$code = $this->_checkThenNext(self::T_IntegerConstant);
$name = $this->_checkThenNext(self::T_Identifier);
$this->_checkThenNext('(');
$tokens = $this->shared_tokens;
if($this->token == self::T_Prop)
$tokens = array_merge($tokens, $this->_parsePropTokens());
$req_fields = $this->_parseFields(function()
{ return $this->_nextIf(')'); }
);
$rsp_fields = $this->_parseFields(function()
{ return $this->_nextIf(self::T_End); }
);
$req = new mtgMetaPacket($code, "RPC_REQ_$name");
$req->setFields($req_fields);
$rsp = new mtgMetaPacket($code, "RPC_RSP_$name");
$rsp->setFields($rsp_fields);
$rpc = new mtgMetaRPC("RPC_$name", $code, $req, $rsp, $tokens);
$this->_addUnit(new mtgMetaInfoUnit($this->file, $rpc));
}
private function _parsePropTokens()
{
$new_line = ord("\n");
$prop_tokens = array();
while(true)
{
if($this->token != self::T_Prop)
break;
$name = ltrim($this->attribute, '@');
$this->_next();
$value = null;
if($this->token == ord(':'))
{
while(true)
{
$this->_next(false/*don't skip new line*/);
if($this->token == $new_line ||
$this->token == self::T_Prop)
{
//let's skip it
if($this->token == $new_line)
$this->_next();
break;
}
else
{
$tmp = $this->attribute;
if($this->token == self::T_StringConstant)
$tmp = "\"$tmp\"";
if($value === null)
$value = '';
$value .= $tmp;
}
}
}
if($value && substr($value, 0, 1) == '{')
{
$json = json_decode($value);
if($json === null)
{
--$this->line; //hack for more precise reporting
$this->_error("bad json");
}
}
$this->_validatePropToken($name, $value);
$prop_tokens[$name] = $value;
}
return $prop_tokens;
}
private function _validatePropToken($name, $value)
{
if(!isset($this->config['valid_tokens']) ||
!is_array($this->config['valid_tokens']))
return;
if(!in_array($name, $this->config['valid_tokens']))
{
--$this->line; //hack for more precise reporting
throw new Exception("Unknown token '$name'");
}
}
private function _symbol()
{
return substr($this->source, $this->cursor, 1);
}
private function _next($skip_newlines = true)
{
$this->__next($skip_newlines);
//for debug
//var_dump("NEXT " . $this->token . " " . $this->attribute);
//debug_print_backtrace(0, 1);
}
private function __next($skip_newlines = true)
{
while(true)
{
$c = $this->_symbol();
//NOTE: dealing with PHP's types juggling
if($c === false || $c === '')
$c = -1;
$this->token = ord($c);
++$this->cursor;
$this->attribute = $c;
switch($c)
{
case -1: $this->cursor--; $this->token = self::T_EOF; return;
case ' ': case "\r": case "\t": break;
case "\n": $this->line++; if($skip_newlines) break; else return;
case '{': case '}': case '(': case ')': case '[': case ']': case '|': return;
case ',': case ':': case ';': case '=': return;
case '.':
if(!ctype_digit($this->_symbol())) return;
$this->_error("floating point constant can't start with .");
break;
case '"':
$this->attribute = "";
while($this->_symbol() != '"')
{
if(ord($this->_symbol()) < ord(' ') && ord($this->_symbol()) >= 0)
$this->_error("illegal character in string constant");
if($this->_symbol() == '\\')
{
$this->cursor++;
switch($this->_symbol())
{
case 'n': $this->attribute .= "\n"; $this->cursor++; break;
case 't': $this->attribute .= "\t"; $this->cursor++; break;
case 'r': $this->attribute .= "\r"; $this->cursor++; break;
case '"': $this->attribute .= '"'; $this->cursor++; break;
case '\\': $this->attribute .= '\\'; $this->cursor++; break;
default: $this->_error("unknown escape code in string constant"); break;
}
}
else // printable chars + UTF-8 bytes
{
$this->attribute .= $this->_symbol();
$this->cursor++;
}
}
$this->token = self::T_StringConstant;
$this->cursor++;
return;
case '`':
$this->attribute = "";
while($this->_symbol() != '`')
{
$this->attribute .= $this->_symbol();
$this->cursor++;
}
$this->token = self::T_RawStringConstant;
$this->cursor++;
return;
case '/':
if($this->_symbol() == '/')
{
$this->cursor++;
while($this->_symbol() !== false && $this->_symbol() != "\n") $this->cursor++;
break;
}
case '#':
while($this->_symbol() !== false && $this->_symbol() != "\n") $this->cursor++;
break;
case '@':
$start = $this->cursor - 1;
while(ctype_alnum($this->_symbol()) || $this->_symbol() == '_')
$this->cursor++;
$this->token = self::T_Prop;
$this->attribute = substr($this->source, $start, $this->cursor - $start);
return;
//fall thru
default:
if(ctype_alpha($c))
{
//collect all chars of an identifier
$start = $this->cursor - 1;
while(ctype_alnum($this->_symbol()) || $this->_symbol() == '_')
$this->cursor++;
$this->attribute = substr($this->source, $start, $this->cursor - $start);
if(isset($this->idltypes[$this->attribute]))
{
$this->token = $this->idltypes[$this->attribute];
return;
}
if($this->attribute == "true" || $this->attribute == "false")
{
$this->token = self::T_IntegerConstant;
return;
}
//check for declaration keywords:
if($this->attribute == "struct") { $this->token = self::T_Struct; return; }
if($this->attribute == "interface") { $this->token = self::T_Interface; return; }
if($this->attribute == "enum") { $this->token = self::T_Enum; return; }
if($this->attribute == "RPC") { $this->token = self::T_RPC; return; }
if($this->attribute == "end") { $this->token = self::T_End; return; }
if($this->attribute == "extends") { $this->token = self::T_Extends; return; }
if($this->attribute == "implements") { $this->token = self::T_Implements; return; }
if($this->attribute == "func") { $this->token = self::T_Func; return; }
//if not it's a user defined identifier
$this->token = self::T_Identifier;
return;
}
else if(ctype_digit($c) || $c == '-')
{
$start = $this->cursor - 1;
while(ctype_digit($this->_symbol())) $this->cursor++;
if($this->_symbol() == '.')
{
$this->cursor++;
while(ctype_digit($this->_symbol())) $this->cursor++;
// see if this float has a scientific notation suffix. Both JSON
// and C++ (through strtod() we use) have the same format:
if($this->_symbol() == 'e' || $this->_symbol() == 'E')
{
$this->cursor++;
if($this->_symbol() == '+' || $this->_symbol() == '-') $this->cursor++;
while(ctype_digit($this->_symbol())) $this->cursor++;
}
$this->token = self::T_FloatConstant;
}
else
$this->token = self::T_IntegerConstant;
$this->attribute = substr($this->source, $start, $this->cursor - $start);
return;
}
$this->_error("illegal character '$c'");
}
}
}
private function _nextIf($t)
{
if(is_string($t))
$t = ord($t);
$yes = $t === $this->token;
if($yes)
$this->_next();
return $yes;
}
private function _checkThenNext($t)
{
if(is_string($t))
$t = ord($t);
if($t !== $this->token)
{
$this->_error("expecting '" . $this->_toStr($t) . "' instead got '" . $this->_toStr($this->token) . "'");
}
$attr = $this->attribute;
$this->_next();
return $attr;
}
private function _toStr($t)
{
if($t < 1000)
return chr($t);
return $this->token_strs[$t];
}
private function _error($msg)
{
throw new Exception($msg . "(token: {$this->token}, attr: {$this->attribute})");
}
}
class mtgMetaParsedModule
{
public $file;
public $units = array();
public $includes = array();
function __construct($file)
{
$this->file = $file;
}
function addUnit(mtgMetaInfoUnit $unit)
{
$this->units[$unit->object->getId()] = $unit;
}
function findUnit($id)
{
if(isset($this->units[$id]))
return $this->units[$id];
foreach($this->includes as $include)
{
if(isset($include->units[$id]))
return $include->units[$id];
}
return null;
}
function addInclude(mtgMetaParsedModule $include)
{
$this->includes[] = $include;
}
}
function mtg_parse_meta(array $meta_srcs, $valid_tokens = null, $inc_path = null)
{
if($inc_path === null)
{
//let's autodetect include path
$inc_path = array();
foreach($meta_srcs as $src)
{
if(is_dir($src))
$inc_path[] = $src;
else if(is_file($src))
$inc_path[] = dirname($src);
}
}
$meta_parser = new mtgMetaInfoParser(
array(
'include_path' => $inc_path,
'valid_tokens' => $valid_tokens
)
);
$meta = new mtgMetaInfo();
foreach($meta_srcs as $src)
mtg_load_meta($meta, $meta_parser, $src);
return $meta;
}
function mtg_load_meta(mtgMetaInfo $meta, mtgMetaInfoParser $meta_parser, $dir_or_file)
{
$files = array();
if(is_dir($dir_or_file))
$files = mtg_find_meta_files($dir_or_file);
else if(is_file($dir_or_file))
$files[] = $dir_or_file;
else
throw new Exception("Bad meta source '$dir_or_file'");
foreach($files as $file)
$meta_parser->parse($meta, $file);
}
function mtg_find_meta_files($dir)
{
$items = scandir($dir);
if($items === false)
throw new Exception("Directory '$dir' is invalid");
$files = array();
foreach($items as $item)
{
if($item[0] == '.')
continue;
if(strpos($item, ".meta") !== (strlen($item)-5))
continue;
$file = $dir . '/' . $item;
if(is_file($file) && !is_dir($file))
$files[] = $file;
}
return $files;
}