Added extension passing in mtg_parse_meta

.gitea/workflows/build_composer.yaml

@@ -0,0 +1,30 @@
+
+
+name: Publish PHP Package
+
+on:
+  push:
+    tags:
+      - 'v*'
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Get tag name
+      run: echo "TAG=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
+
+    - name: zip and send
+      run: |
+        ls -la
+        apt-get update -y
+        apt-get install -y zip
+        cd ../
+        zip -r ${{ gitea.event.repository.name }}.zip ${{ gitea.event.repository.name }} -x '*.git*'
+        curl -v \
+        --user composer-pbl:${{ secrets.COMPOSER_PSWD }} \
+        --upload-file ${{ gitea.event.repository.name }}.zip \
+        https://git.bit5.ru/api/packages/bit/composer?version=${{ env.TAG }}              

CHANGELOG.md

@@ -0,0 +1,8 @@
+## v3.10.0
+- Adding validation of type refs pointing to named functions
+
+## v3.9.0
+- Starting to experiment with new metagen parser v2.0a
+
+## v3.8.0
+- Adding initial support for service type

metagen.inc.php

@@ -1,19 +1,25 @@
 <?php
 
 interface mtgType
-{}
+{
+  function getName();
+}
+
+interface mtgScope
+{
+  function findSymbol($name);
+}
 
 class mtgTypeRef implements mtgType
 {
-  private static $all = array();
+  private static $unresolved = array();
 
   private $name;
-  private $file;
-  private $line;
-  private $meta;
+  private $scope;
   private $resolved;
+  private $origin;
 
-  function __construct($name_or_resolved, mtgMetaInfo $meta = null, $file = '', $line= 0)
+  function __construct($name_or_resolved, mtgScope $scope = null, mtgOrigin $origin = null)
   {
     if(is_object($name_or_resolved))
     {
@@ -22,19 +28,36 @@ class mtgTypeRef implements mtgType
       $this->resolved = $name_or_resolved;
     }
     else
+    {
       $this->name = $name_or_resolved;
+      if(!$scope)
+        throw new Exception("Scope is not set");
 
-    $this->meta = $meta;
-    $this->file = $file;
-    $this->line = $line;
+      self::$unresolved[] = $this;
+    }
 
-    self::$all[] = $this;
+    $this->scope = $scope;
+    $this->origin = $origin ?? new mtgOrigin();
+
+  }
+
+  function getName()
+  {
+    if($this->resolved)
+      return ''.$this->resolved;
+    else
+      return $this->name;
   }
 
   static function checkAllResolved()
   {
-    foreach(self::$all as $ref)
-      $ref->resolve();
+    while(sizeof(self::$unresolved) > 0)
+    {
+      $ref = array_shift(self::$unresolved);
+      $resolved = $ref->resolve();
+      if($resolved instanceof mtgMetaFunc && $resolved->getName())
+        throw new Exception("Invalid resolving of named function as type: " . $resolved->getName() . " at " . $ref->origin);
+    }
   }
 
   function resolve()
@@ -42,28 +65,45 @@ class mtgTypeRef implements mtgType
     if($this->resolved)
       return $this->resolved;
 
-    $u = $this->meta->findUnit($this->name, false/*non strict*/);
-    if($u)
+    $symb = $this->scope->findSymbol($this->name);
+    if($symb)
     {
-      $this->resolved = $u->object;
+      $this->resolved = $symb;
       return $this->resolved;
     }
     else
-    {
-      throw new Exception("{$this->file}@{$this->line} : Symbol '{$this->name}' not found");
-    }
+      throw new Exception("{$this->origin} : Symbol '{$this->name}' not found");
   } 
+
+  function __toString()
+  {
+    return $this->getName();
+  }
 }
 
 class mtgMetaInfoUnit
 {
-  public $file;
-  public $object;
+  public ?mtgMetaParsedModule $module = null;
+  public string $file;
+  public mtgMetaUnit $object;
 
-  function __construct($f, mtgMetaUnit $o)
+  /**
+   * @param string|mtgMetaParsedModule $file_or_module
+   */
+  function __construct($file_or_module, mtgMetaUnit $obj)
   {
-    $this->file = $f;
-    $this->object = $o;
+    if($file_or_module instanceof mtgMetaParsedModule)
+    {
+      $this->module = $file_or_module;
+      $this->file = $file_or_module->file;
+    }
+    else if(is_string($file_or_module))
+    {
+      $this->module = null;
+      $this->file = $file_or_module;
+    }
+
+    $this->object = $obj;
   }
 }
 
@@ -78,32 +118,51 @@ class mtgMetaInfo
 
   function addUnit(mtgMetaInfoUnit $unit)
   {
-    if(isset($this->units[$unit->object->getId()]))
-      throw new Exception("Meta info unit '{$unit->object->getId()}' already defined in file '{$this->units[$unit->object->getId()]->file}'");
+    if(isset($this->units[$unit->object->getMetaId()]))
+      throw new Exception("Meta info unit '{$unit->object->getMetaId()}' already defined in file '{$this->units[$unit->object->getMetaId()]->file}'");
 
-    $this->units[$unit->object->getId()] = $unit;
+    $this->units[$unit->object->getMetaId()] = $unit;
   }
 
-  function getUnits()
+  function delUnit(mtgMetaInfoUnit $unit)
+  {
+    unset($this->units[$unit->object->getMetaId()]);
+  }
+
+  function getUnits() : array
   {
     return $this->units;
   }
 
-  function findUnit($id, $strict = true)
+  function findUnit($id) : ?mtgMetaInfoUnit
   {
     if(isset($this->units[$id]))
       return $this->units[$id];
-    else if($strict)
-      throw new Exception("Unit '$id' not found");
+    return null;
+  }
+
+  function getUnit($id) : mtgMetaInfoUnit
+  {
+    if(isset($this->units[$id]))
+      return $this->units[$id];
+    throw new Exception("Unit '$id' not found");
+  }
+
+  function validate()
+  {
+    foreach($this->units as $u)
+      $u->object->validate($this);
   }
 }
 
 abstract class mtgMetaUnit
 {
-  abstract function getId();
+  abstract function getMetaId();
 
   protected $tokens = array();
 
+  protected ?mtgOrigin $origin;
+
   function getTokens()
   {
     return $this->tokens;
@@ -114,12 +173,17 @@ abstract class mtgMetaUnit
     $this->tokens = $tokens;
   }
 
-  function setToken($name, $val)
+  function setToken(string $name, $val)
   {
     $this->tokens[$name] = $val;
   }
 
-  function getToken($name)
+  function delToken(string $name)
+  {
+    unset($this->tokens[$name]);
+  }
+
+  function getToken(string $name)
   {
     return $this->hasToken($name) ? $this->tokens[$name] : null; 
   }
@@ -128,9 +192,38 @@ abstract class mtgMetaUnit
   {
     return array_key_exists($name, $this->tokens);
   }
+
+  function setOrigin(mtgOrigin $origin)
+  {
+    $this->origin = $origin;
+  }
+
+  function getOrigin() : ?mtgOrigin
+  {
+    return $this->origin;
+  }
+
+  abstract function validate(mtgMetaInfo $meta);
 }
 
-class mtgUserType extends mtgMetaUnit implements mtgType
+class mtgOrigin
+{
+  public $file;
+  public $line;
+
+  function __construct($file = '', $line = 0)
+  {
+    $this->file = $file;
+    $this->line = $line;
+  }
+
+  function __toString()
+  {
+    return "{$this->file}@{$this->line}";
+  }
+}
+
+abstract class mtgUserType extends mtgMetaUnit implements mtgType
 {
   protected $name;
 
@@ -139,6 +232,11 @@ class mtgUserType extends mtgMetaUnit implements mtgType
     $this->name = $name;
   }
 
+  function getMetaId()
+  {
+    return $this->name;
+  }
+
   function getName()
   {
     return $this->name;
@@ -149,15 +247,12 @@ class mtgUserType extends mtgMetaUnit implements mtgType
     $this->name = $name;
   }
 
-  function getId()
-  {
-    return $this->name;
-  }
-
   function getClassId()
   {
-    //TODO: use more flexible schema, maybe get rid of this method 
-    //NOTE: using crc28 actually, leaving some extra reserved space
+    if($this->hasToken('class_id'))
+      return $this->getToken('class_id');
+
+    //TODO: migrate to crc32 from crc28
     return crc32($this->name) & 0xFFFFFFF;
   }
 
@@ -172,14 +267,49 @@ class mtgMetaStruct extends mtgUserType
   protected $fields = array();
   protected $funcs = array();
   protected $parent = null;
+  protected $implements = array();
 
-  function __construct($name, $fields = array(), mtgTypeRef $parent = null, $tokens = array())
+  function __construct($name, array $fields = array(), mtgTypeRef $parent = null, array $tokens = array(), array $implements = array())
   {
     parent::__construct($name);
 
     $this->setFields($fields);
     $this->parent = $parent;
     $this->tokens = $tokens;
+    $this->implements = $implements;
+  }
+
+  function validate(mtgMetaInfo $meta)
+  {
+    $parent = $this->getParent();
+    while($parent != null)
+    {
+      foreach($this->fields as $name => $_)
+        if($parent->hasField($name))
+          throw new Exception("{$this->origin} : Parent struct '{$parent->getName()}' already has field '{$name}'");
+
+      foreach($this->funcs as $name => $_)
+        if($parent->hasFunc($name))
+          throw new Exception("{$this->origin} : Parent struct '{$parent->getName()}' already has func '{$name}'");
+
+      $parent = $parent->getParent();
+    }
+
+    $s = new SplObjectStorage();
+    foreach($this->getImplements() as $imp)
+    {
+      if(!($imp instanceof mtgMetaInterface))
+        throw new Exception("{$this->origin} : Not an interface '{$imp}'");
+
+      if($s->contains($imp))
+        throw new Exception("{$this->origin} : Duplicate interface reference '{$imp->getName()}'");
+      $s->attach($imp);
+    }
+
+    foreach($this->funcs as $func)
+    {
+      $func->validate($meta);
+    }
   }
 
   function getParent()
@@ -187,6 +317,17 @@ class mtgMetaStruct extends mtgUserType
     return $this->parent ? $this->parent->resolve() : null;
   }
 
+  function getImplements()
+  {
+    if(!$this->implements)
+      return array();
+
+    $imps = array();
+    foreach($this->implements as $imp)
+      $imps[] = $imp->resolve();
+    return $imps;
+  }
+
   function getFields()
   {
     return $this->fields;
@@ -272,9 +413,17 @@ class mtgMetaInterface extends mtgUserType
     $this->tokens = $tokens;
   }
 
+  function validate(mtgMetaInfo $meta) {}
+
   function getImplements()
   {
-    return $this->parent ? $this->parent->resolve() : null;
+    if(!$this->implements)
+      return array();
+
+    $imps = array();
+    foreach($this->implements as $imp)
+      $imps[] = $imp->resolve();
+    return $imps;
   }
 
   function getFuncs()
@@ -314,6 +463,14 @@ class mtgMetaFunc extends mtgMetaUnit implements mtgType
     $this->name = $name;
   }
 
+  function validate(mtgMetaInfo $meta)
+  {}
+
+  function getMetaId()
+  {
+    return $this->name;
+  }
+
   function __toString()
   {
     $str = "func ";
@@ -343,11 +500,6 @@ class mtgMetaFunc extends mtgMetaUnit implements mtgType
     return $this->name;
   }
 
-  function getId()
-  {
-    return $this->name;
-  }
-
   function getArgs()
   {
     return $this->args;
@@ -396,7 +548,9 @@ class mtgMetaRPC extends mtgMetaUnit
     $this->tokens = $tokens;
   }
 
-  function getId()
+  function validate(mtgMetaInfo $meta) {}
+
+  function getMetaId()
   {
     return $this->code;
   }
@@ -427,6 +581,122 @@ class mtgMetaRPC extends mtgMetaUnit
   }
 }
 
+class mtgMetaService extends mtgMetaUnit implements mtgScope
+{
+  private $name;
+  private $parent_scope;
+  private $rpcs = array();
+  private $events = array();
+  private $user_types = array();
+
+  function __construct($name, mtgScope $parent_scope, array $tokens = array())
+  {
+    $this->name = $name;
+    $this->parent_scope = $parent_scope;
+    $this->tokens = $tokens;
+  }
+
+  function validate(mtgMetaInfo $meta) {}
+
+  function getName()
+  {
+    return $this->name;
+  }
+
+  function getMetaId()
+  {
+    return $this->name;
+  }
+
+  function findSymbol($name)
+  {
+    if(isset($this->user_types[$name]))
+      return $this->user_types[$name];
+
+    return $this->parent_scope->findSymbol($name);
+  }
+
+  function getRPCs() : array
+  {
+    return $this->rpcs;
+  }
+
+  function addRPC(mtgMetaRPC $rpc)
+  {
+    foreach($this->rpcs as $name => $_rpc)
+    {
+      if($name == $rpc->getName())
+        throw new Exception("Service '{$this->name}' already has RPC '{$rpc->getName()}'");
+
+      if($rpc->getNumericCode() == $_rpc->getNumericCode())
+        throw new Exception("Service '{$this->name}' already has RPC '{$_rpc->getName()}' with code '{$rpc->getNumericCode()}'");
+    }
+
+    $this->rpcs[$rpc->getName()] = $rpc;
+  }
+
+  function hasRPC($name)
+  {
+    return isset($this->rpcs[$name]);
+  }
+
+  function getRPC($name)
+  {
+    if(!isset($this->rpcs[$name]))
+      throw new Exception("No such RPC '$name'");
+    return $this->rpcs[$name];
+  }
+
+  function getUserTypes() : array
+  {
+    return $this->user_types;
+  }
+
+  function addUserType(mtgUserType $utype)
+  {
+    if($this->hasRPC($utype->getName()) || $this->hasUserType($utype->getName()))
+      throw new Exception("Service '{$this->name}' already has type '{$utype->getName()}'");
+
+    $this->user_types[$utype->getName()] = $utype;
+  }
+
+  function hasUserType($name)
+  {
+    return isset($this->user_types[$name]);
+  }
+
+  function getUserType($name)
+  {
+    if(!isset($this->user_types[$name]))
+      throw new Exception("No such user type '$name'");
+    return $this->user_types[$name];
+  }
+
+  function addEvent(mtgMetaStruct $evt)
+  {
+    $this->events[] = $evt;
+    $this->addUserType($evt);
+  }
+
+  function getEvents() : array
+  {
+    return $this->events;
+  }
+
+  function getClassId()
+  {
+    if($this->hasToken('class_id'))
+      return $this->getToken('class_id');
+
+    return crc32($this->name);
+  }
+
+  function __toString()
+  {
+    return $this->name;
+  }
+}
+
 class mtgBuiltinType implements mtgType
 {
   private $name;
@@ -554,13 +824,18 @@ class mtgMultiType implements mtgType
     return $vals;
   }
 
-  function __toString()
+  function getName()
   {
     $str = '';
     foreach($this->getValues() as $val)
       $str .= $val . ';';
     return $str;
   }
+
+  function __toString()
+  {
+    return $this->getName();
+  }
 }
 
 class mtgArrType implements mtgType
@@ -572,11 +847,16 @@ class mtgArrType implements mtgType
     $this->value = $value;
   }
 
-  function __toString()
+  function getName()
   {
     return $this->getValue() . '[]';
   }
 
+  function __toString()
+  {
+    return $this->getName();
+  }
+
   function getValue()
   {
     return $this->value->resolve();
@@ -634,6 +914,7 @@ class mtgMetaField
   {
     return $this->hasToken($name) ? $this->tokens[$name] : null; 
   }
+  
 }
 
 class mtgMetaPacket extends mtgMetaStruct
@@ -673,6 +954,8 @@ class mtgMetaEnum extends mtgUserType
 {
   private $values = array();
 
+  function validate(mtgMetaInfo $meta) {}
+
   function addValue($value_name, $value)
   {
     if(isset($this->values[$value_name]))
@@ -687,7 +970,7 @@ class mtgMetaEnum extends mtgUserType
   function calcOrValue($or_keys)
   {
     $res = 0;
-    foreach ($or_keys as $value_name)
+    foreach($or_keys as $value_name)
     {
       if(!isset($this->values[$value_name]))
         throw new Exception("Enum '{$this->name}' has no value '{$value_name}'");
@@ -707,6 +990,18 @@ class mtgMetaEnum extends mtgUserType
   {
     return $this->values;
   }
+
+  function override(mtgMetaEnum $other)
+  {
+    $this->tokens = array_merge($this->tokens, $other->getTokens());
+    $this->values = array_merge($this->values, $other->getValues());
+  }
+
+  function replace(mtgMetaEnum $other)
+  {
+    $this->tokens = $other->getTokens();
+    $this->values = $other->getValues();
+  }
 } 
 
 function mtg_get_all_fields(mtgMetaStruct $struct)

parser.inc.php

@@ -1,810 +1,60 @@
 <?php
+require_once(__DIR__ . '/parser1.inc.php');
+require_once(__DIR__ . '/parser2.inc.php');
 
-class mtgMetaInfoParser
+define('MTG_PARSER_TYPE_1', "1.0");
+define('MTG_PARSER_TYPE_2', "2.0a");
+
+interface mtgIMetaInfoParser
 {
-  private $config = array();
-  private $current_meta;
-  private $parsed_files = array();
-  private $file_stack = array();
-  private $file = "";
-  private $source = "";
-  private $cursor = 0;
-  private $line = 0;
-  private $token = "";
-  private $attribute = "";
-  private $idltypes = array();
-  private $token_strs = array();
+  function parse(mtgMetaInfo $meta, string $file);
+}
 
-  const T_EOF                = 1001;
-  const T_StringConstant     = 1002;
-  const T_IntegerConstant    = 1003;
-  const T_FloatConstant      = 1004;
-  const T_Enum               = 1005;
-  const T_RPC                = 1006;
-  const T_End                = 1007;
-  const T_Identifier         = 1008;
-  const T_Struct             = 1009;
-  const T_Prop               = 1010;
-  const T_Extends            = 1011;
-  const T_Func               = 1012;
-  const T_RawStringConstant  = 1013;
-  const T_Interface          = 1014;
-  const T_string             = 1020;
-  const T_uint32             = 1021;
-  const T_int32              = 1022;
-  const T_uint16             = 1023;
-  const T_int16              = 1024;
-  const T_uint8              = 1025;
-  const T_int8               = 1026;
-  const T_float              = 1027;
-  const T_uint64             = 1028;
-  const T_int64              = 1029;
-  const T_bool               = 1030;
-  const T_blob               = 1031;
+class mtgMetaParsedModule implements mtgScope
+{
+  public string $file;
+  public $units = array();
+  public $includes = array();
 
-  function __construct($config = array())
+  function __construct(string $file)
   {
-    $this->config = $config;
-    if(!isset($this->config['include_path']))
-      $this->config['include_path'] = array('.');
-
-    $this->idltypes = array(
-      "string" => self::T_string,
-      "uint32" => self::T_uint32,
-      "int32" => self::T_int32,
-      "uint16" => self::T_uint16,
-      "int16" => self::T_int16,
-      "uint8" => self::T_uint8,
-      "int8" => self::T_int8,
-      "float" => self::T_float,
-      "double" => self::T_float,
-      "uint64" => self::T_uint64,
-      "int64" => self::T_int64,
-      "bool" => self::T_bool,
-      "blob" => self::T_blob,
-    );
-    $this->token_strs = array_flip($this->idltypes);
-    $this->token_strs[self::T_EOF] = '<EOF>';
-    $this->token_strs[self::T_StringConstant] = '<StringConstant>';
-    $this->token_strs[self::T_RawStringConstant] = '<RawStringConstant>';
-    $this->token_strs[self::T_IntegerConstant] = '<IntegerConstant>';
-    $this->token_strs[self::T_FloatConstant] = '<FloatConstant>';
-    $this->token_strs[self::T_Enum] = '<enum>';
-    $this->token_strs[self::T_RPC] = '<RPC>';
-    $this->token_strs[self::T_End] = '<end>';
-    $this->token_strs[self::T_Identifier] = '<Identifier>';
-    $this->token_strs[self::T_Struct] = '<struct>';
-    $this->token_strs[self::T_Interface] = '<interface>';
-    $this->token_strs[self::T_Prop] = '<@prop>';
-    $this->token_strs[self::T_Extends] = '<extends>';
-    $this->token_strs[self::T_Func] = '<func>';
-  }
-
-  function parse(mtgMetaInfo $meta, $raw_file)
-  {
-    $this->current_meta = $meta;
-
-    $file = realpath($raw_file);
-    if($file === false)
-      throw new Exception("No such file '$raw_file'");
-
-    $this->_parse($file);
-
-    mtgTypeRef::checkAllResolved();
-  }
-
-  private function _parse($file)
-  {
-    if(isset($this->parsed_files[$file]))
-      return;
-    $this->parsed_files[$file] = true;
-    $this->file_stack[] = $file;
-    $source = file_get_contents($file);
-    $is_php = false;
-
-    try
-    {
-      if($source === false)
-        throw new Exception("Could not read file '$file'");
-
-      //PHP include
-      if(strpos($source, '<?php') === 0)
-      {
-        include_once($file);
-        $is_php = true;
-      }
-      else
-      {
-        self::resolveIncludes($source, $this->config['include_path'], array($this, '_parse'));
-      }
-    }
-    catch(Exception $e)
-    {
-      throw new Exception(end($this->file_stack) . " : " . $e->getMessage());
-    }
-
-    array_pop($this->file_stack);
-
-    if($is_php)
-      return;
-
     $this->file = $file;
-    $this->source = $source;
-    $this->cursor = 0;
-    $this->line = 1;
-
-    try
-    {
-      $this->_next();
-      while($this->token != self::T_EOF)
-      {
-        //echo "TOKEN : " . $this->token . " " . $this->attribute . " " . $this->line . "\n";
-
-        if($this->token == self::T_Enum)
-          $this->_parseEnum();
-        else if($this->token == self::T_Struct)
-          $this->_parseStruct();
-        else if($this->token == self::T_Interface)
-          $this->_parseInterface();
-        else if($this->token == self::T_Func)
-          $this->_parseFreeFunc();
-        else if($this->token == self::T_RPC)
-          $this->_parseRPC();
-        else
-          $this->_error("unexpected token ('" . $this->_toStr($this->token) . "' " . $this->attribute . ")");
-      }
-    }
-    catch(Exception $e)
-    {
-      throw new Exception("$file@{$this->line} : " .  $e->getMessage() . " " . $e->getTraceAsString());
-    }
   }
 
-  private function _parseType($can_be_multi = false)
+  function addUnit(mtgMetaInfoUnit $unit)
   {
-    $types = array();
+    $this->units[$unit->object->getMetaId()] = $unit;
+  }
 
-    while(true)
+  function findSymbol($name)
+  {
+    $unit = $this->findUnit($name);
+    if(!$unit)
+      return null;
+    return $unit->object;
+  }
+
+  function findUnit($id)
+  {
+    if(isset($this->units[$id]))
+      return $this->units[$id];
+
+    foreach($this->includes as $include)
     {
-      $type = null;
-      
-      if($this->token == self::T_Func)
-      {
-        $func_type = $this->_parseFuncType();
-        $type = new mtgTypeRef($func_type);
-      }
-      else if($this->token == self::T_Identifier)
-      {
-        $type_name = $this->_parseDotName();
-        $type = new mtgTypeRef($type_name, $this->current_meta, $this->file, $this->line);
-      }
-      else
-      {
-        $type_name = $this->attribute;
-        $type = new mtgTypeRef(new mtgBuiltinType($type_name));
-        $this->_next();
-      }
-
-      if($this->token == ord('['))
-      {
-        $this->_next();
-        $this->_checkThenNext(']');
-        $type = new mtgTypeRef(new mtgArrType($type));
-      }
-      $types[] = $type;
-
-      if(!$can_be_multi)
-        break;
-
-      if($this->token != ord(','))
-        break;
-      $this->_next();
+      if(isset($include->units[$id]))
+        return $include->units[$id];
     }
 
-    if(sizeof($types) > 1)
-      return new mtgTypeRef(new mtgMultiType($types));
-    else
-      return $types[0];
+    return null;
   }
 
-  private function _parseFuncType()
+  function addInclude(mtgMetaParsedModule $include)
   {
-    $ftype = new mtgMetaFunc('');
-
-    $this->_next();
-
-    $this->_checkThenNext('(');
-
-    $c = 0;
-    while(true)
-    {
-      if($this->token == ord(')'))
-      {
-        $this->_next();
-        break;
-      }
-      else if($c > 0)
-      {
-        $this->_checkThenNext(',');
-      }
-
-      $arg_type = $this->_parseType();
-      $c++;
-      $arg = new mtgMetaField("_$c", $arg_type);
-      $ftype->addArg($arg);
-    }
-
-    if($this->token == ord(':'))
-    {
-      $this->_next();
-      $ret_type = $this->_parseType(true/*can be multi-type*/);
-      $ftype->setReturnType($ret_type);
-    }
-    return $ftype;
-  }
-
-  static function resolveIncludes(&$text, array $include_paths, $callback)
-  {
-    $result = array();
-    $lines = explode("\n", $text);
-    foreach($lines as $line)
-    {
-      if(preg_match('~^#include\s+(\S+)~', $line, $m))
-      {
-        self::processInclude($m[1], $include_paths, $callback);
-        $result[] = "";
-      }
-      else
-        $result[] = $line;
-    }
-    $text = implode("\n", $result);
-  }
-
-  static function processInclude($include, array $include_paths, $callback)
-  {
-    $file = false;
-    foreach($include_paths as $include_path)
-    {
-      $file = realpath($include_path . "/" . $include);
-      if($file !== false)
-        break;
-    }
-
-    if($file === false)
-      throw new Exception("#include {$include} can't be resolved(include path is '". implode(':', $include_paths) . "')");
-
-    call_user_func_array($callback, array($file));
-  }
-
-  private function _parseEnumOrValues()
-  {
-    $values = array();
-    while(true)
-    {
-      if($this->token == self::T_Identifier)          
-      {
-        $values[] = $this->attribute;
-        $this->_next();
-        if($this->token != ord('|'))
-          break;
-        else
-          $this->_next();
-      }
-      else
-        break;
-    }
-
-    return $values;
-  }
-
-  private function _parseEnum()
-  {
-    $this->_next();
-
-    $name = $this->_parseDotName();
-
-    $enum = new mtgMetaEnum($name);
-    $this->current_meta->addUnit(new mtgMetaInfoUnit($this->file, $enum));
-
-    if($this->token == self::T_Prop)
-    {
-      $enum->setTokens($this->_parsePropTokens());
-    }
-
-    $or_values = array();
-    while(true)
-    {
-      if($this->_nextIf(self::T_End))
-        break;
-      $key = $this->_checkThenNext(self::T_Identifier);
-      $this->_checkThenNext('=');
-      if($this->token == self::T_Identifier)
-      {
-        $or_values[$key] = $this->_parseEnumOrValues();
-      }
-      else
-      {
-        $value = $this->_checkThenNext(self::T_IntegerConstant);
-        $enum->addValue($key, $value);
-      }
-    }
-    $enum->addOrValues($or_values);
-  }
-
-  private function _parseFields($next_doer)
-  {
-    $flds = array();
-
-    while(true)
-    {
-      if($next_doer())
-        break;
-
-      if($this->token == self::T_Identifier)
-      {
-        $name = $this->attribute;
-        $this->_next();
-        $this->_checkThenNext(':');
-
-        if($this->token == self::T_Identifier || 
-          $this->token == self::T_Func ||
-          ($this->token >= self::T_string && $this->token <= self::T_blob))
-        {
-          $type = $this->_parseType();
-
-          $fld = new mtgMetaField($name, $type);
-
-          if($this->token == self::T_Prop)
-            $fld->setTokens($this->_parsePropTokens());
-
-          $flds[] = $fld;
-        }
-        else
-          $this->_error("type expected");
-      }
-      else
-        $this->_error("unexpected fields token");
-    }
-
-    return $flds;
-  }
-
-  private function _parseFuncs()
-  {
-    $end_token = self::T_End;
-
-    $funcs = array();
-
-    while(true)
-    {
-      $fn = $this->_parseFunc();
-      $funcs[] = $fn;
-
-      if($this->token == $end_token)
-      {
-        $this->_next();
-        break;
-      }
-      
-      $this->_next();
-    }
-
-    return $funcs;
-  }
-
-  private function _parseDotName()
-  {
-    $dot_name = '';
-
-    while(true)
-    {
-      if($this->token != self::T_Identifier)
-        $this->_error("unexpected name token");
-      
-      $dot_name .= $this->attribute;
-      $this->_next();
-      if($this->token != ord('.'))
-        break;
-      $dot_name .= '.';
-      $this->_next();
-    }
-
-    return $dot_name;
-  }
-
-  private function _parseFunc()
-  {
-    $name = $this->_parseDotName();
-    $fn = new mtgMetaFunc($name);
-
-    $this->_checkThenNext('(');
-    if($this->token == self::T_Prop)
-      $fn->setTokens($this->_parsePropTokens());
-    $args = $this->_parseFields(function() 
-      { return $this->_nextIf(')'); } 
-    );
-    $fn->setArgs($args);
-
-    $ret_type = null;
-    if($this->token == ord(':'))
-    {
-      $this->_next();
-      if($this->token == self::T_Identifier ||
-         $this->token == self::T_Func ||
-        ($this->token >= self::T_string && $this->token <= self::T_bool))
-      {
-        $ret_type = $this->_parseType(true/*can be multi-type*/);
-        $fn->setReturnType($ret_type);
-      }
-      else
-        $this->_error("unexpected func type token");
-    }
-
-    return $fn;
-  }
-
-  private function _parseFreeFunc()
-  {
-    $this->_next();
-    $fn = $this->_parseFunc();
-    $this->current_meta->addUnit(new mtgMetaInfoUnit($this->file, $fn));
-  }
-
-  private function _parseStruct()
-  {
-    $this->_next();
-    $name = $this->_parseDotName();
-
-    $parent = null;
-    if($this->token == self::T_Extends)
-    {
-      $this->_next();
-      $parent_name = $this->_checkThenNext(self::T_Identifier);
-      $parent = new mtgTypeRef($parent_name, $this->current_meta, $this->file, $this->line);
-    }
-
-    $s = new mtgMetaStruct($name, array(), $parent);
-    $this->current_meta->addUnit(new mtgMetaInfoUnit($this->file, $s));
-
-    if($this->token == self::T_Prop)
-      $s->setTokens($this->_parsePropTokens());
-
-    $seen_funcs = false;
-    $flds = $this->_parseFields(
-      function() use(&$seen_funcs) 
-      { 
-        if($this->_nextIf(self::T_End))
-          return true;
-        if($this->_nextIf(self::T_Func)) 
-        {
-          $seen_funcs = true;
-          return true;
-        }
-      }
-    );
-    foreach($flds as $fld)
-      $s->addField($fld);
-
-    if($seen_funcs)
-    {
-      $funcs = $this->_parseFuncs();
-      foreach($funcs as $fn)
-        $s->addFunc($fn);
-    }
-  }
-
-  private function _parseInterface()
-  {
-    $this->_next();
-    $name = $this->_parseDotName();
-
-    $s = new mtgMetaInterface($name);
-    $this->current_meta->addUnit(new mtgMetaInfoUnit($this->file, $s));
-
-    if($this->token == self::T_Prop)
-      $s->setTokens($this->_parsePropTokens());
-
-    $this->_next();
-    $funcs = $this->_parseFuncs();
-    foreach($funcs as $fn)
-      $s->addFunc($fn);
-  }
-
-  private function _parseRPC()
-  {
-    $this->_next();
-    $code = $this->_checkThenNext(self::T_IntegerConstant);
-    $name = $this->_checkThenNext(self::T_Identifier);
-    $this->_checkThenNext('(');
-
-    $tokens = array();
-    if($this->token == self::T_Prop)
-      $tokens = $this->_parsePropTokens();
-
-    $req_fields = $this->_parseFields(function() 
-      { return $this->_nextIf(')'); } 
-    );
-    $rsp_fields = $this->_parseFields(function() 
-      { return $this->_nextIf(self::T_End); } 
-    );
-
-    $req = new mtgMetaPacket($code, "RPC_REQ_$name");
-    $req->setFields($req_fields);
-    $rsp = new mtgMetaPacket($code, "RPC_RSP_$name");
-    $rsp->setFields($rsp_fields);
-
-    $rpc = new mtgMetaRPC("RPC_$name", $code, $req, $rsp, $tokens);
-    $this->current_meta->addUnit(new mtgMetaInfoUnit($this->file, $rpc));
-  }
-
-  private function _parsePropTokens()
-  {
-    $new_line = ord("\n");
-
-    $prop_tokens = array();
-
-    while(true)
-    {
-      if($this->token != self::T_Prop)
-        break;
-
-      $name = ltrim($this->attribute, '@');
-      $this->_next();
-
-      $value = null;
-      if($this->token == ord(':'))
-      {
-        while(true)
-        {
-          $this->_next(false/*don't skip new line*/);
-          if($this->token == $new_line || 
-             $this->token == self::T_Prop)
-          {
-            //let's skip it
-            if($this->token == $new_line)
-              $this->_next();
-            break;
-          }
-          else
-          {
-            $tmp = $this->attribute; 
-            if($this->token == self::T_StringConstant)
-              $tmp = "\"$tmp\"";
-            if($value === null)
-              $value = '';
-            $value .= $tmp;
-          }
-        }
-      }
-      if($value && substr($value, 0, 1) == '{')
-      {
-        $json = json_decode($value);
-        if($json === null)
-        {
-          --$this->line; //hack for more precise reporting
-          $this->_error("bad json");
-        }
-      } 
-      $this->_validatePropToken($name, $value);
-
-      $prop_tokens[$name] = $value;
-    }
-    return $prop_tokens;
-  }
-
-  private function _validatePropToken($name, $value)
-  {
-    if(!isset($this->config['valid_tokens']) || 
-       !is_array($this->config['valid_tokens']))
-      return;
-
-    if(!in_array($name, $this->config['valid_tokens']))
-    {
-      --$this->line; //hack for more precise reporting
-      throw new Exception("Unknown token '$name'");
-    }
-  }
-
-  private function _symbol()
-  {
-    return substr($this->source, $this->cursor, 1);
-  }
-
-  private function _next($skip_newlines = true)
-  {
-    $this->__next($skip_newlines);
-    //for debug
-    //var_dump("NEXT " . $this->token . " " . $this->attribute);
-    //debug_print_backtrace(0, 1);
-  }
-
-  private function __next($skip_newlines = true)
-  {
-    while(true)
-    {
-      $c = $this->_symbol();
-      //NOTE: dealing with PHP's types juggling
-      if($c === false || $c === '')
-        $c = -1;
-
-      $this->token = ord($c);
-      ++$this->cursor;
-      $this->attribute = $c;
-
-      switch($c)
-      {
-        case -1: $this->cursor--; $this->token = self::T_EOF; return;
-        case ' ': case "\r": case "\t": break;
-        case "\n": $this->line++; if($skip_newlines) break; else return;
-        case '{': case '}': case '(': case ')': case '[': case ']': case '|': return;
-        case ',': case ':': case ';': case '=': return;
-        case '.':
-          if(!ctype_digit($this->_symbol())) return;
-          $this->_error("floating point constant can't start with .");
-          break;
-        case '"':
-          $this->attribute = "";
-          while($this->_symbol() != '"') 
-          {
-            if(ord($this->_symbol()) < ord(' ') && ord($this->_symbol()) >= 0)
-              $this->_error("illegal character in string constant");
-            if($this->_symbol() == '\\') 
-            {
-              $this->cursor++;
-              switch($this->_symbol()) 
-              {
-                case 'n':  $this->attribute .= "\n"; $this->cursor++; break;
-                case 't':  $this->attribute .= "\t"; $this->cursor++; break;
-                case 'r':  $this->attribute .= "\r"; $this->cursor++; break;
-                case '"': $this->attribute .= '"'; $this->cursor++; break;
-                case '\\': $this->attribute .= '\\'; $this->cursor++; break;
-                default: $this->_error("unknown escape code in string constant"); break;
-              }
-            }
-            else // printable chars + UTF-8 bytes
-            {
-              $this->attribute .= $this->_symbol();
-              $this->cursor++;
-            }
-          }
-          $this->token = self::T_StringConstant;
-          $this->cursor++;
-          return;
-
-        case '`':
-          $this->attribute = "";
-          while($this->_symbol() != '`') 
-          {
-            $this->attribute .= $this->_symbol();
-            $this->cursor++;
-          }
-          $this->token = self::T_RawStringConstant;
-          $this->cursor++;
-          return;
-
-        case '/':
-          if($this->_symbol() == '/') 
-          {
-            $this->cursor++;
-            while($this->_symbol() !== false && $this->_symbol() != "\n") $this->cursor++;
-            break;
-          }
-
-        case '#':
-          while($this->_symbol() !== false && $this->_symbol() != "\n") $this->cursor++;
-          break;
-
-        case '@':
-          $start = $this->cursor - 1;
-          while(ctype_alnum($this->_symbol()) || $this->_symbol() == '_')
-            $this->cursor++;
-          $this->token = self::T_Prop;
-          $this->attribute = substr($this->source, $start, $this->cursor - $start);
-          return;
-
-        //fall thru
-        default:
-
-          if(ctype_alpha($c))
-          {
-            //collect all chars of an identifier
-            $start = $this->cursor - 1;
-            while(ctype_alnum($this->_symbol()) || $this->_symbol() == '_')
-              $this->cursor++;
-            $this->attribute = substr($this->source, $start, $this->cursor - $start);
-
-            if(isset($this->idltypes[$this->attribute]))
-            {
-              $this->token = $this->idltypes[$this->attribute];
-              return;
-            }
-
-            if($this->attribute == "true" || $this->attribute == "false") 
-            {
-              $this->token = self::T_IntegerConstant;
-              return;
-            }
-
-            //check for declaration keywords:
-            if($this->attribute == "struct")    { $this->token = self::T_Struct;    return; }
-            if($this->attribute == "interface") { $this->token = self::T_Interface; return; }
-            if($this->attribute == "enum")      { $this->token = self::T_Enum;      return; }
-            if($this->attribute == "RPC")       { $this->token = self::T_RPC;       return; }
-            if($this->attribute == "end")       { $this->token = self::T_End;       return; }
-            if($this->attribute == "extends")   { $this->token = self::T_Extends;   return; }
-            if($this->attribute == "func")      { $this->token = self::T_Func;      return; }
-
-            //if not it's a user defined identifier
-            $this->token = self::T_Identifier;
-            return;
-          }
-          else if(ctype_digit($c) || $c == '-')
-          {
-            $start = $this->cursor - 1;
-            while(ctype_digit($this->_symbol())) $this->cursor++;
-            if($this->_symbol() == '.')
-            {
-              $this->cursor++;
-              while(ctype_digit($this->_symbol())) $this->cursor++;
-              // see if this float has a scientific notation suffix. Both JSON
-              // and C++ (through strtod() we use) have the same format:
-              if($this->_symbol() == 'e' || $this->_symbol() == 'E') 
-              {
-                $this->cursor++;
-                if($this->_symbol() == '+' || $this->_symbol() == '-') $this->cursor++;
-                while(ctype_digit($this->_symbol())) $this->cursor++;
-              }
-              $this->token = self::T_FloatConstant;
-            }
-            else
-              $this->token = self::T_IntegerConstant;
-            $this->attribute = substr($this->source, $start, $this->cursor - $start);
-            return;
-          }
-
-          $this->_error("illegal character '$c'");
-      }
-    }
-  }
-
-  private function _nextIf($t)
-  {
-    if(is_string($t))
-      $t = ord($t);
-    $yes = $t === $this->token;
-    if($yes)
-      $this->_next();
-    return $yes;
-  }
-
-  private function _checkThenNext($t)
-  {
-    if(is_string($t))
-      $t = ord($t);
-    if($t !== $this->token)
-    {
-      $this->_error("expecting '" . $this->_toStr($t) . "' instead got '" . $this->_toStr($this->token) . "'");
-    }
-
-    $attr = $this->attribute;
-    $this->_next();
-    return $attr;
-  }
-
-  private function _toStr($t)
-  {
-    if($t < 1000)
-      return chr($t);
-    return $this->token_strs[$t];
-  }
-
-  private function _error($msg)
-  {
-    throw new Exception($msg . "(token: {$this->token}, attr: {$this->attribute})");
+    $this->includes[] = $include;
   }
 }
 
-function mtg_parse_meta(array $meta_srcs, $valid_tokens = null, $inc_path = null)
+function mtg_parse_meta(array $meta_srcs, $valid_tokens = null, $inc_path = null, $version = null, string $extension = '.meta')
 {
   if($inc_path === null)
   {
@@ -819,33 +69,60 @@ function mtg_parse_meta(array $meta_srcs, $valid_tokens = null, $inc_path = null
     }
   }
 
-  $meta_parser = new mtgMetaInfoParser(
-    array(
-      'include_path' => $inc_path, 
-      'valid_tokens' => $valid_tokens
-    )
-  );
+  $meta_parser = null;
+
+  if($version === null || $version === MTG_PARSER_TYPE_1)
+    $meta_parser = new mtgMetaInfoParser(
+        array(
+          'include_path' => $inc_path, 
+          'valid_tokens' => $valid_tokens
+        )
+      );
+  else if($version === MTG_PARSER_TYPE_2)
+    $meta_parser = new mtgMetaInfoParser2(
+        array(
+          'include_path' => $inc_path, 
+          'valid_tokens' => $valid_tokens
+        )
+      );
+  else
+    throw new Exception("Unknown meta parser version: $version");
+
   $meta = new mtgMetaInfo();
+
   foreach($meta_srcs as $src)
-    mtg_load_meta($meta, $meta_parser, $src);
+    mtg_load_meta($meta, $meta_parser, $src, $extension);
+
+  $meta->validate();
+
+  mtgTypeRef::checkAllResolved();
+
   return $meta;
 }
 
-function mtg_load_meta(mtgMetaInfo $meta, mtgMetaInfoParser $meta_parser, $dir_or_file)
+function mtg_load_meta(
+  mtgMetaInfo $meta, mtgIMetaInfoParser $meta_parser, 
+  string $dir_or_file, string $extension = '.meta'
+)
 {
-  $files = array();
-  if(is_dir($dir_or_file))
-    $files = mtg_find_meta_files($dir_or_file);
-  else if(is_file($dir_or_file))
-    $files[] = $dir_or_file;
-  else
-    throw new Exception("Bad meta source '$dir_or_file'");
-
+  $files = mtg_resolve_files($dir_or_file, $extension);
   foreach($files as $file)
     $meta_parser->parse($meta, $file);
 }
 
-function mtg_find_meta_files($dir)
+function mtg_resolve_files(string $dir_or_file, string $extension = '.meta') : array
+{
+  $files = array();
+  if(is_dir($dir_or_file))
+    $files = mtg_find_meta_files($dir_or_file, $extension);
+  else if(is_file($dir_or_file))
+    $files[] = $dir_or_file;
+  else
+    throw new Exception("Bad meta source '$dir_or_file'");
+  return $files;
+}
+
+function mtg_find_meta_files($dir, $extension = '.meta')
 {
   $items = scandir($dir);
   if($items === false)
@@ -857,7 +134,7 @@ function mtg_find_meta_files($dir)
     if($item[0] == '.')
       continue;
 
-    if(strpos($item, ".meta") !== (strlen($item)-5))
+    if(strpos($item, $extension) !== (strlen($item)-strlen($extension)))
       continue;
 
     $file = $dir . '/' . $item;

parser1.inc.php

@@ -0,0 +1,973 @@
+<?php
+
+class mtgMetaInfoParser implements mtgIMetaInfoParser
+{
+  const T_NonOrdMark         = 1000; //marks start of non 'ord(..)' values
+  const T_EOF                = 1001;
+  const T_StringConstant     = 1002;
+  const T_IntegerConstant    = 1003;
+  const T_FloatConstant      = 1004;
+  const T_Enum               = 1005;
+  const T_RPC                = 1006;
+  const T_End                = 1007;
+  const T_UserSymbol         = 1008;
+  const T_Struct             = 1009;
+  const T_Prop               = 1010;
+  const T_Extends            = 1011;
+  const T_Func               = 1012;
+  const T_RawStringConstant  = 1013;
+  const T_Interface          = 1014;
+  const T_Implements         = 1015;
+  const T_MinType            = 1019; //built-in types begin mark
+  const T_string             = 1020;
+  const T_uint32             = 1021;
+  const T_int32              = 1022;
+  const T_uint16             = 1023;
+  const T_int16              = 1024;
+  const T_uint8              = 1025;
+  const T_int8               = 1026;
+  const T_float              = 1027;
+  const T_uint64             = 1028;
+  const T_int64              = 1029;
+  const T_bool               = 1030;
+  const T_blob               = 1031;
+  const T_Service            = 1032;
+  const T_MaxType            = 1033; //built-in types end mark
+
+  private array $config = array();
+  private mtgMetaInfo $current_meta;
+  private array $parsed_files = array();
+  private array $file_stack = array();
+  private ?mtgMetaParsedModule $module = null;
+  private string $file = "";
+  private string $source = "";
+  private int $cursor_pos = 0;
+  private string $cursor_char = '';
+  private int $line = 0;
+  //TODO: setting it an 'int' type makes PHPStan produce many 
+  //      false positives
+  //token numeric identifier 
+  private $T = 0;
+  //token extra string value which depends on concrete T 
+  private string $T_value = "";
+  /** @var array<string,int>*/
+  private $symbol2T = array();
+  /** @var array<int,string>*/
+  private $T2descr = array();
+  private array $shared_tokens = array();
+  private array $scopes = array();
+
+  function __construct($config = array())
+  {
+    $this->_initTables();
+
+    self::_addDefaultTokens($config);
+
+    $this->config = $config;
+    if(!isset($this->config['include_path']))
+      $this->config['include_path'] = array('.');
+  }
+
+  private function _initTables()
+  {
+    $this->symbol2T = [
+      "string" => self::T_string,
+      "uint32" => self::T_uint32,
+      "int32" => self::T_int32,
+      "uint16" => self::T_uint16,
+      "int16" => self::T_int16,
+      "uint8" => self::T_uint8,
+      "int8" => self::T_int8,
+      "float" => self::T_float,
+      "double" => self::T_float,
+      "uint64" => self::T_uint64,
+      "int64" => self::T_int64,
+      "bool" => self::T_bool,
+      "blob" => self::T_blob,
+
+      "true" => self::T_IntegerConstant,
+      "false" => self::T_IntegerConstant,
+
+      "struct" => self::T_Struct,
+      "interface" => self::T_Interface,
+      "enum" => self::T_Enum,
+      "RPC" => self::T_RPC,
+      "service" => self::T_Service,
+      "end" => self::T_End,
+      "extends" => self::T_Extends,
+      "implements" => self::T_Implements,
+      "func" => self::T_Func,
+    ];
+
+    $this->T2descr = array_flip($this->symbol2T);
+    $this->T2descr[self::T_EOF] = '<EOF>';
+    $this->T2descr[self::T_StringConstant] = '<StringConstant>';
+    $this->T2descr[self::T_RawStringConstant] = '<RawStringConstant>';
+    $this->T2descr[self::T_IntegerConstant] = '<IntegerConstant>';
+    $this->T2descr[self::T_FloatConstant] = '<FloatConstant>';
+    $this->T2descr[self::T_Enum] = '<enum>';
+    $this->T2descr[self::T_RPC] = '<RPC>';
+    $this->T2descr[self::T_Service] = '<Service>';
+    $this->T2descr[self::T_End] = '<end>';
+    $this->T2descr[self::T_UserSymbol] = '<Identifier>';
+    $this->T2descr[self::T_Struct] = '<struct>';
+    $this->T2descr[self::T_Interface] = '<interface>';
+    $this->T2descr[self::T_Prop] = '<@prop>';
+    $this->T2descr[self::T_Extends] = '<extends>';
+    $this->T2descr[self::T_Implements] = '<implements>';
+    $this->T2descr[self::T_Func] = '<func>';
+  }
+
+  private static function _addDefaultTokens(array &$config)
+  {
+    if(!isset($config['valid_tokens']))
+      $config['valid_tokens'] = array();
+
+    $config['valid_tokens'][] = 'class_id';
+    $config['valid_tokens'][] = 'shared_tokens';
+    $config['valid_tokens'][] = 'enum_override';
+    $config['valid_tokens'][] = 'enum_replace';
+  }
+
+  function parse(mtgMetaInfo $meta, string $raw_file)
+  {
+    $this->current_meta = $meta;
+
+    $file = realpath($raw_file);
+    if($file === false)
+      throw new Exception("No such file '$raw_file'");
+
+    $this->_parse($file);
+  }
+
+  private function _parse(string $file)
+  {
+    if(isset($this->parsed_files[$file]))
+      return;
+    $module = new mtgMetaParsedModule($file);
+    $this->parsed_files[$file] = $module;
+    $this->file_stack[] = $file;
+    $source = file_get_contents($file);
+
+    try
+    {
+      if($source === false)
+        throw new Exception("Could not read file '$file'");
+
+      $this->_resolveIncludes($module, $source);
+    }
+    catch(Exception $e)
+    {
+      throw new Exception(end($this->file_stack) . " : " . $e->getMessage());
+    }
+
+    array_pop($this->file_stack);
+
+    $this->module = $module;
+    $this->file = $file;
+    $this->source = $source;
+    $this->line = 1;
+    $this->cursor_pos = -1;
+    $this->_cursorNext();
+    $this->shared_tokens = array();
+
+    try
+    {
+      $this->_nextT();
+      while($this->T != self::T_EOF)
+      {
+        if($this->T == self::T_Prop)
+          $this->_parseSharedTokens($this->_parsePropTokens());
+        else if($this->T == self::T_Enum)
+          $this->_parseEnum();
+        else if($this->T == self::T_Struct)
+          $this->_parseStruct();
+        else if($this->T == self::T_Interface)
+          $this->_parseInterface();
+        else if($this->T == self::T_Func)
+          $this->_parseFreeFunc();
+        else if($this->T == self::T_RPC)
+          $this->_parseRPC();
+        else if($this->T == self::T_Service)
+          $this->_parseService();
+        else
+          $this->_error("Unexpected symbol ('" . $this->_toStr($this->T) . "' " . $this->T_value . ")");
+      }
+    }
+    catch(Exception $e)
+    {
+      throw new Exception("$file@{$this->line} : " .  $e->getMessage() . " " . $e->getTraceAsString());
+    }
+  }
+
+  private function _parseInclude(mtgMetaParsedModule $module, string $file)
+  {
+    $this->_parse($file);
+
+    $module->addInclude($this->parsed_files[$file]);
+  }
+
+  private function _parseSharedTokens(array $tokens)
+  {
+    if(!isset($tokens['shared_tokens']))
+      return;
+
+    $this->shared_tokens = json_decode($tokens['shared_tokens'], true);
+    if(!is_array($this->shared_tokens))
+      $this->_error("Invalid 'shared_tokens' formant, invalid JSON");
+  }
+
+  private function _parseType(bool $can_be_multi = false)
+  {
+    $types = array();
+
+    while(true)
+    {
+      $type = null;
+      
+      if($this->T == self::T_Func)
+      {
+        $origin = new mtgOrigin($this->file, $this->line);
+        $func_type = $this->_parseFuncType();
+        $type = new mtgTypeRef($func_type, $this->_scope(), $origin);
+      }
+      else if($this->T == self::T_UserSymbol)
+      {
+        $origin = new mtgOrigin($this->file, $this->line);
+        $type_name = $this->_parseDotName();
+        $type = new mtgTypeRef($type_name, $this->_scope(), $origin);
+      }
+      else
+      {
+        $origin = new mtgOrigin($this->file, $this->line);
+        $type_name = $this->T_value;
+        $type = new mtgTypeRef(new mtgBuiltinType($type_name), $this->_scope(), $origin);
+        $this->_nextT();
+      }
+
+      if($this->T == ord('['))
+      {
+        $origin = new mtgOrigin($this->file, $this->line);
+        $this->_nextT();
+        $this->_checkThenNext(ord(']'));
+        $type = new mtgTypeRef(new mtgArrType($type), $this->_scope(), $origin);
+      }
+      $types[] = $type;
+
+      if(!$can_be_multi)
+        break;
+
+      if($this->T != ord(','))
+        break;
+      $this->_nextT();
+    }
+
+    if(sizeof($types) > 1)
+      return new mtgTypeRef(new mtgMultiType($types), $this->_scope(), new mtgOrigin($this->file, $this->line));
+    else
+      return $types[0];
+  }
+
+  private function _parseFuncType()
+  {
+    $ftype = new mtgMetaFunc('');
+
+    $this->_nextT();
+
+    $this->_checkThenNext(ord('('));
+
+    $c = 0;
+    while(true)
+    {
+      if($this->T == ord(')'))
+      {
+        $this->_nextT();
+        break;
+      }
+      else if($c > 0)
+      {
+        $this->_checkThenNext(ord(','));
+      }
+
+      $arg_type = $this->_parseType();
+      $c++;
+      $arg = new mtgMetaField("_$c", $arg_type);
+      $ftype->addArg($arg);
+    }
+
+    if($this->T == ord(':'))
+    {
+      $this->_nextT();
+      $ret_type = $this->_parseType(true/*can be multi-type*/);
+      $ftype->setReturnType($ret_type);
+    }
+    return $ftype;
+  }
+
+  private function _resolveIncludes(mtgMetaParsedModule $module, string &$text)
+  { 
+    $include_paths = $this->config['include_path'];
+
+    $result = array();
+    $lines = explode("\n", $text);
+    foreach($lines as $line)
+    {
+      if(preg_match('~^#include\s+(\S+)~', $line, $m))
+      {
+        $this->_processInclude($module, $m[1], $include_paths);
+        $result[] = "";
+      }
+      else
+        $result[] = $line;
+    }
+    $text = implode("\n", $result);
+  }
+
+  private function _processInclude(mtgMetaParsedModule $module, string $include, array $include_paths)
+  {
+    $file = false;
+    foreach($include_paths as $include_path)
+    {
+      $file = realpath($include_path . "/" . $include);
+      if($file !== false)
+        break;
+    }
+
+    if($file === false)
+      throw new Exception("#include {$include} can't be resolved(include path is '". implode(':', $include_paths) . "')");
+
+    $this->_parseInclude($module, $file);
+  }
+
+  private function _parseEnumOrValues()
+  {
+    $values = array();
+    while(true)
+    {
+      if($this->T == self::T_UserSymbol)          
+      {
+        $values[] = $this->T_value;
+        $this->_nextT();
+        if(!$this->_nextIf(ord('|')))
+          break;
+      }
+      else
+        break;
+    }
+
+    return $values;
+  }
+
+  private function _parseEnum($is_global = true)
+  {
+    $this->_nextT();
+    $name = $this->_parseDotName();
+
+    $enum = new mtgMetaEnum($name);
+    $tokens = $this->shared_tokens;
+    if($this->T == self::T_Prop)
+      $tokens = array_merge($tokens, $this->_parsePropTokens());
+    $enum->setTokens($tokens);
+
+    $or_values = array();
+    while(true)
+    {
+      if($this->_nextIf(self::T_End))
+        break;
+      $key = $this->T_value;
+      $this->_checkThenNext(self::T_UserSymbol);
+      $this->_checkThenNext(ord('='));
+      if($this->T == self::T_UserSymbol)
+      {
+        $or_values[$key] = $this->_parseEnumOrValues();
+      }
+      else
+      {
+        $value = $this->T_value;
+        $this->_checkThenNext(self::T_IntegerConstant);
+        $enum->addValue($key, $value);
+      }
+    }
+    $enum->addOrValues($or_values);
+
+    //NOTE: special case for enums when we allow to 'override' the original one, 
+    //      with additional values 
+    if($enum->hasToken('enum_override'))
+    {
+      if(!$is_global)
+        $this->_error("Override supported for global enums only");
+
+      $existing = $this->current_meta->findUnit($enum->getMetaId());
+      if(!$existing)
+        throw new Exception("Not found '{$name}' enum to override values");
+      if(!($existing->object instanceof mtgMetaEnum))
+        throw new Exception("Not an enum struct '{$name}'");
+
+      $existing->object->override($enum);
+    }
+    //NOTE: special case for enums when we allow to 'replace' the original one, 
+    //      with additional values 
+    else if($enum->hasToken('enum_replace'))
+    {
+      if(!$is_global)
+        $this->_error("Replace supported for global enums only");
+
+      $existing = $this->current_meta->findUnit($enum->getMetaId());
+      if(!$existing)
+        throw new Exception("Not found '{$name}' enum to replace values");
+      if(!($existing->object instanceof mtgMetaEnum))
+        throw new Exception("Not an enum struct '{$name}'");
+
+      $existing->object->replace($enum);
+    }
+    else if($is_global)
+      $this->_addUnit(new mtgMetaInfoUnit($this->module, $enum));
+
+    return $enum;
+  }
+
+  static private function _isBuiltinType(int $t) : bool
+  {
+    return $t > self::T_MinType && $t < self::T_MaxType;
+  }
+
+  private function _pushScope(mtgScope $scope)
+  {
+    $this->scopes[] = $scope;
+  }
+
+  private function _popScope()
+  {
+    array_shift($this->scopes);
+  }
+
+  private function _scope() : mtgScope
+  {
+    if(!$this->scopes)
+      return $this->module;
+
+    return $this->scopes[count($this->scopes)-1];
+  }
+
+  private function _parseFields(callable $next_doer)
+  {
+    $flds = array();
+
+    while(true)
+    {
+      if($next_doer())
+        break;
+
+      if($this->T == self::T_UserSymbol)
+      {
+        $name = $this->T_value;
+        $this->_nextT();
+        $this->_checkThenNext(ord(':'));
+
+        if($this->T == self::T_UserSymbol || 
+          $this->T == self::T_Func ||
+          self::_isBuiltinType($this->T))
+        {
+          $type = $this->_parseType();
+
+          $fld = new mtgMetaField($name, $type);
+
+          if($this->T == self::T_Prop)
+            $fld->setTokens($this->_parsePropTokens());
+
+          $flds[] = $fld;
+        }
+        else
+          $this->_error("Type expected");
+      }
+      else
+        $this->_error("Unexpected fields symbol");
+    }
+
+    return $flds;
+  }
+
+  private function _parseFuncs()
+  {
+    $end_token = self::T_End;
+
+    $funcs = array();
+
+    while(true)
+    {
+      $fn = $this->_parseFunc();
+      $funcs[] = $fn;
+
+      if($this->T == $end_token)
+      {
+        $this->_nextT();
+        break;
+      }
+      
+      $this->_nextT();
+    }
+
+    return $funcs;
+  }
+
+  private function _parseDotName() : string
+  {
+    $dot_name = '';
+
+    while(true)
+    {
+      if($this->T != self::T_UserSymbol)
+        $this->_error("Unexpected name symbol");
+      
+      $dot_name .= $this->T_value;
+      $this->_nextT();
+      if($this->T != ord('.'))
+        break;
+      $dot_name .= '.';
+      $this->_nextT();
+    }
+
+    return $dot_name;
+  }
+
+  private function _parseFunc() : mtgMetaFunc
+  {
+    $name = $this->_parseDotName();
+    $fn = new mtgMetaFunc($name);
+
+    $this->_checkThenNext(ord('('));
+    if($this->T == self::T_Prop)
+      $fn->setTokens($this->_parsePropTokens());
+    $args = $this->_parseFields(function() 
+      { return $this->_nextIf(ord(')')); } 
+    );
+    $fn->setArgs($args);
+
+    $ret_type = null;
+    if($this->T == ord(':'))
+    {
+      $this->_nextT();
+      if($this->T == self::T_UserSymbol ||
+         $this->T == self::T_Func ||
+        self::_isBuiltinType($this->T))
+      {
+        $ret_type = $this->_parseType(true/*can be multi-type*/);
+        $fn->setReturnType($ret_type);
+      }
+      else
+        $this->_error("Unexpected func type");
+    }
+
+    return $fn;
+  }
+
+  private function _addUnit(mtgMetaInfoUnit $unit)
+  {
+    $this->current_meta->addUnit($unit);
+    $this->module->addUnit($unit);
+  }
+
+  private function _parseFreeFunc()
+  {
+    $this->_nextT();
+    $fn = $this->_parseFunc();
+    $fn->setTokens(array_merge($this->shared_tokens, $fn->getTokens()));
+    $this->_addUnit(new mtgMetaInfoUnit($this->module, $fn));
+  }
+
+  private function _parseStruct()
+  {
+    $this->_nextT();
+    $struct_origin = new mtgOrigin($this->file, $this->line);
+    $name = $this->_parseDotName();
+
+    $parent = null;
+    if($this->T == self::T_Extends)
+    {
+      $this->_nextT();
+      $origin = new mtgOrigin($this->file, $this->line);
+      $parent_name = $this->_parseDotName();
+      $parent = new mtgTypeRef($parent_name, $this->_scope(), $origin);
+    }
+
+    $implements = array();
+    if($this->T == self::T_Implements)
+    {
+      do 
+      {
+        $this->_nextT();
+        $origin = new mtgOrigin($this->file, $this->line);
+        $if_name = $this->_parseDotName();
+        $implements[] = new mtgTypeRef($if_name, $this->_scope(), $origin);
+      } while($this->T == ord(','));
+    }
+
+    $s = new mtgMetaStruct($name, array(), $parent, array(), $implements);
+    $s->setOrigin($struct_origin);
+    $this->_addUnit(new mtgMetaInfoUnit($this->module, $s));
+
+    $tokens = $this->shared_tokens;
+    if($this->T == self::T_Prop)
+      $tokens = array_merge($tokens, $this->_parsePropTokens());
+    $s->setTokens($tokens);
+
+    $seen_funcs = false;
+    $flds = $this->_parseFields(
+      function() use(&$seen_funcs) 
+      { 
+        if($this->_nextIf(self::T_End))
+          return true;
+        if($this->_nextIf(self::T_Func)) 
+        {
+          $seen_funcs = true;
+          return true;
+        }
+      }
+    );
+    foreach($flds as $fld)
+      $s->addField($fld);
+
+    if($seen_funcs)
+    {
+      $funcs = $this->_parseFuncs();
+      foreach($funcs as $fn)
+        $s->addFunc($fn);
+    }
+  }
+
+  private function _parseInterface()
+  {
+    $this->_nextT();
+    $name = $this->_parseDotName();
+
+    $s = new mtgMetaInterface($name);
+    $this->_addUnit(new mtgMetaInfoUnit($this->module, $s));
+
+    $tokens = $this->shared_tokens;
+    if($this->T == self::T_Prop)
+      $tokens = array_merge($tokens, $this->_parsePropTokens());
+    $s->setTokens($tokens);
+
+    if($this->T !== self::T_End)
+    {
+      $this->_nextT();
+      $funcs = $this->_parseFuncs();
+      foreach($funcs as $fn)
+        $s->addFunc($fn);
+    }
+    else
+      $this->_nextT();
+  }
+
+  private function _parseRPC($is_global = true)
+  {
+    $this->_nextT();
+    $code = $this->T_value;
+    $this->_checkThenNext(self::T_IntegerConstant);
+    $name = $this->_parseDotName();
+    $this->_checkThenNext(ord('('));
+
+    $tokens = $this->shared_tokens;
+    if($this->T == self::T_Prop)
+      $tokens = array_merge($tokens, $this->_parsePropTokens());
+
+    $req_fields = $this->_parseFields(function() 
+      { return $this->_nextIf(ord(')')); } 
+    );
+    $rsp_fields = $this->_parseFields(function() 
+      { return $this->_nextIf(self::T_End); } 
+    );
+
+    $req = new mtgMetaPacket($code, $is_global ? "RPC_REQ_$name" : "Request");
+    $req->setFields($req_fields);
+    $rsp = new mtgMetaPacket($code, $is_global ? "RPC_RSP_$name" : "Response");
+    $rsp->setFields($rsp_fields);
+
+    $rpc = new mtgMetaRPC($is_global ? "RPC_$name" : $name, $code, $req, $rsp, $tokens);
+
+    if($is_global)
+      $this->_addUnit(new mtgMetaInfoUnit($this->module, $rpc));
+
+    return $rpc;
+  }
+
+  private function _parseService()
+  {
+    $this->_nextT();
+    $name = $this->_parseDotName();
+
+    $service = new mtgMetaService($name, $this->_scope());
+    $this->_pushScope($service);
+
+    $tokens = $this->shared_tokens;
+    if($this->T == self::T_Prop)
+      $tokens = array_merge($tokens, $this->_parsePropTokens());
+    $service->setTokens($tokens);
+
+    while(true)
+    {
+      if($this->_nextIf(self::T_End))
+        break;
+      $key = $this->T_value;
+      if($this->T == self::T_RPC)
+        $service->addRPC($this->_parseRPC(false));
+      else if($this->T == self::T_Enum)
+        $service->addUserType($this->_parseEnum(false));
+      else
+        $this->_error("Unsupported type");
+    }
+
+    $this->_popScope();
+
+    $this->_addUnit(new mtgMetaInfoUnit($this->module, $service));
+  }
+
+  private function _parsePropTokens() : array
+  {
+    $prop_tokens = array();
+
+    while(true)
+    {
+      if($this->T != self::T_Prop)
+        break;
+
+      $name = ltrim($this->T_value, '@');
+      $this->_validatePropToken($name);
+      $this->_nextT();
+
+      $value = null;
+      if($this->T == ord(':'))
+      {
+        //let's read the value
+        while(true)
+        {
+          //TODO: The code below is ugly and must be heavily refactored,
+          //      it just tries to be convenient and keep BC: any token property
+          //      value can have almost any kind of symbols excluding new line. 
+          //      In the future we should restrict property values to certain types only
+          $this->_nextT(true/*stop on new line*/);
+
+          if($this->T == ord("\n"))
+          {
+            $this->_nextT();
+            break;
+          }
+          else if($this->T == self::T_Prop)
+          {
+            break;
+          }
+          else
+          {
+            $tmp = $this->T_value; 
+            if($this->T == self::T_StringConstant)
+              $tmp = "\"$tmp\"";
+            if($value === null)
+              $value = '';
+            $value .= $tmp;
+          }
+        }
+      }
+
+      $prop_tokens[$name] = $value;
+    }
+    return $prop_tokens;
+  }
+
+  private function _validatePropToken(string $name)
+  {
+    if(!isset($this->config['valid_tokens']) || 
+       !is_array($this->config['valid_tokens']))
+      return;
+
+    if(!in_array($name, $this->config['valid_tokens']))
+      throw new Exception("Unknown property token '@$name'");
+  }
+
+  private function _nextT(bool $stop_on_new_line = false)
+  {
+    while(true)
+    {
+      $c = $this->cursor_char;
+
+      //setting default values
+      $this->T = ord($c);
+      $this->T_value = $c;
+
+      //NOTE: current 'cursor_pos' is ahead of 'c' by one character
+      $this->_cursorNext();
+
+      switch($c)
+      {
+        case '': $this->T = self::T_EOF; return;
+        case "\n": if($stop_on_new_line) return; else break;
+        case ' ': case "\r": case "\t": break;
+        case '{': case '}': case '(': case ')': case '[': case ']': case '|': return;
+        case ',': case ':': case ';': case '=': return;
+        case '.':
+          if(!ctype_digit($this->cursor_char)) 
+            return;
+          $this->_error("Floating point constant can't start with .");
+          break;
+        case '"':
+          $this->T_value = '';
+          while($this->cursor_char != '"') 
+          {
+            if(ord($this->cursor_char) < ord(' '))
+              $this->_error("Illegal character in string constant");
+            if($this->cursor_char == '\\') 
+            {
+              $this->_cursorNext();
+              switch($this->cursor_char) 
+              {
+                case 'n':  $this->T_value .= "\n"; $this->_cursorNext(); break;
+                case 't':  $this->T_value .= "\t"; $this->_cursorNext(); break;
+                case 'r':  $this->T_value .= "\r"; $this->_cursorNext(); break;
+                case '"': $this->T_value .= '"'; $this->_cursorNext(); break;
+                case '\\': $this->T_value .= '\\'; $this->_cursorNext(); break;
+                default: $this->_error("Unknown escape code in string constant"); break;
+              }
+            }
+            else // printable chars + UTF-8 bytes
+            {
+              $this->T_value .= $this->cursor_char;
+              $this->_cursorNext();
+            }
+          }
+          $this->T = self::T_StringConstant;
+          $this->_cursorNext();
+          return;
+
+        case '`':
+          $this->T_value = '';
+          //TODO: code below is not robust enough
+          while($this->cursor_char != '`') 
+          {
+            $this->T_value .= $this->cursor_char;
+            $this->_cursorNext();
+          }
+          $this->T = self::T_RawStringConstant;
+          $this->_cursorNext();
+          return;
+
+        case '/':
+          if($this->cursor_char == '/') 
+          {
+            $this->_cursorNext();
+            //@phpstan-ignore-next-line
+            while($this->cursor_char != '' && $this->cursor_char != "\n") 
+              $this->_cursorNext();
+            //@phpstan-ignore-next-line
+            break;
+          }
+
+        case '#':
+          while($this->cursor_char != '' && $this->cursor_char != "\n")
+            $this->_cursorNext(); 
+          break;
+
+        case '@':
+          $start = $this->cursor_pos - 1;
+          while(ctype_alnum($this->cursor_char) || $this->cursor_char == '_')
+            $this->_cursorNext(); 
+
+          $this->T = self::T_Prop;
+          $this->T_value = substr($this->source, $start, $this->cursor_pos - $start);
+          return;
+
+        default:
+
+          //symbols
+          if(ctype_alpha($c))
+          {
+            //collect all chars of an identifier
+            $start = $this->cursor_pos - 1;
+            while(ctype_alnum($this->cursor_char) || $this->cursor_char == '_')
+              $this->_cursorNext();
+            $this->T_value = substr($this->source, $start, $this->cursor_pos - $start);
+
+            if(isset($this->symbol2T[$this->T_value]))
+              $this->T = $this->symbol2T[$this->T_value];
+            else //otherwise it's assumed to be a user defined symbol
+              $this->T = self::T_UserSymbol;
+
+            return;
+          }
+          //digits
+          else if(ctype_digit($c) || $c == '-')
+          {
+            $start = $this->cursor_pos - 1;
+            while(ctype_digit($this->cursor_char)) 
+              $this->_cursorNext();
+            if($this->cursor_char == '.')
+            {
+              $this->_cursorNext();
+              while(ctype_digit($this->cursor_char)) 
+                $this->_cursorNext();
+              // see if this float has a scientific notation suffix. Both JSON
+              // and C++ (through strtod() we use) have the same format:
+              //@phpstan-ignore-next-line
+              if($this->cursor_char == 'e' || $this->cursor_char == 'E') 
+              {
+                $this->_cursorNext();
+                if($this->cursor_char == '+' || $this->cursor_char == '-') 
+                  $this->_cursorNext();
+                while(ctype_digit($this->cursor_char)) 
+                  $this->_cursorNext();
+              }
+              $this->T = self::T_FloatConstant;
+            }
+            else
+              $this->T = self::T_IntegerConstant;
+            $this->T_value = substr($this->source, $start, $this->cursor_pos - $start);
+            return;
+          }
+
+          $this->_error("Illegal character '$c'");
+      }
+    }
+  }
+
+  private function _cursorNext()
+  {
+    ++$this->cursor_pos;
+    $this->cursor_char = substr($this->source, $this->cursor_pos, 1);
+    if($this->cursor_char === "\n") 
+      $this->line++;
+    //EOF
+    if($this->cursor_char === '' || 
+      //keeping BC with substr(..) before php 8.0 
+      //@phpstan-ignore-next-line
+      is_bool($this->cursor_char))
+    {
+      $this->cursor_char = '';
+    }
+  }
+
+  private function _nextIf(int $t) : bool
+  {
+    $yes = $t === $this->T;
+    if($yes)
+      $this->_nextT();
+    return $yes;
+  }
+
+  private function _checkThenNext(int $t)
+  {
+    if($t !== $this->T)
+      $this->_error("Expecting '" . $this->_toStr($t) . "' instead got '" . $this->_toStr($this->T) . "'");
+    $this->_nextT();
+  }
+
+  private function _toStr(int $t) : string
+  {
+    if($t < self::T_NonOrdMark)
+      return chr($t);
+    return $this->T2descr[$t];
+  }
+
+  private function _error(string $msg)
+  {
+    throw new Exception($msg . " ('{$this->T_value}', {$this->T})");
+  }
+}
+