Better check for null default values

.gitea/workflows/build_composer.yaml

@@ -0,0 +1,30 @@
+
+
+name: Publish PHP Package
+
+on:
+  push:
+    tags:
+      - 'v*'
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Get tag name
+      run: echo "TAG=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
+
+    - name: zip and send
+      run: |
+        ls -la
+        apt-get update -y
+        apt-get install -y zip
+        cd ../
+        zip -r ${{ gitea.event.repository.name }}.zip ${{ gitea.event.repository.name }} -x '*.git*'
+        curl -v \
+        --user composer-pbl:${{ secrets.COMPOSER_PSWD }} \
+        --upload-file ${{ gitea.event.repository.name }}.zip \
+        https://git.bit5.ru/api/packages/bit/composer?version=${{ env.TAG }}              

CHANGELOG.md

@@ -0,0 +1,24 @@
+## v6.1.0
+- Removed redundant accessor interface declarations in generated slices
+
+## v5.0.0
+- Splitting one huge codegenerated file into multiple ones
+
+## v4.10.0
+- Added @flt_i18n token to support translations
+
+## v4.9.2
+- Added @cs_obsolete token, that mark fields with [Obsolete]. @cs_obsolete:"Comment" will add summary with "Comment" to field
+
+## v4.9.1
+- Fixing nested diffable POD structs being incorrectly marked as clean in GetDiff
+
+## v4.8.2
+- Fixing thread safety possible issue when reading an array of enums 
+- Fixing bug when the target enum array property wouldn't be cleared before reading data 
+
+## v4.8.0
+- Using C# built-in initialization routines instead of MetaIO's methods
+
+## v4.7.0
+- Array comparison generated code now takes bitfields token into account for array entries 

README.md

@@ -2,12 +2,10 @@ This package is used for code generation of C# meta structs using Twig templates
 
 Usage example:
 
-    $twig = \metagen_cs\get_twig();
-    file_put_contents('bundle.cs', 
-      $twig->render("codegen_bundle.twig", 
-        [
-         'namespace' => 'BitGames.Autogen',
-         'meta' => get_meta()
-        ]
-      )
-    );
+    $output = \metagen_cs\codegen(null, get_meta(), 
+    [
+      'namespace' => 'BitGames.Autogen'
+    ]);
+    foreach($output as $name => $text)
+      file_put_contents($name, $text); 
+

composer.json

@@ -5,7 +5,7 @@
     "require": {
         "php": ">=7.4",
         "twig/twig" : "v3.4.3",
-        "bit/metagen" : "^v2.0.2"
+        "bit/metagen" : "^v3.0.0"
     },
     "autoload": {
         "files": [

src/codegen.inc.php

@@ -2,7 +2,37 @@
 namespace metagen_cs;
 use Exception;
 
-function get_twig(array $inc_path = [])
+function codegen(?string $cache_dir, \mtgMetaInfo $meta, array $options = []) : array
+{
+  $twig = get_twig();
+
+  if(!empty($cache_dir))
+    $twig->setCache($cache_dir);
+
+  $twig->addGlobal('meta', $meta);
+
+  $options['meta'] = $meta;
+  if(!isset($options['namespace']))
+    $options['namespace'] = 'BitGames.Autogen';
+
+  $sliced_units = slice_units($meta->getUnits(), 50);
+
+  $output = array();
+
+  foreach($sliced_units as $slice_idx => $slice_units)
+  {
+    $slice_options = $options;
+    $slice_options['slice_idx'] = $slice_idx;
+    $slice_options['slice_units'] = $slice_units;
+    $output['types_'.$slice_idx.'.cs'] = $twig->render('codegen_types_slice.twig', $slice_options);
+  }
+
+  $output['factory.cs'] = $twig->render('codegen_factory.twig', $options);
+
+  return $output;
+}
+
+function get_twig(array $inc_path = []) : \Twig\Environment
 {
   array_unshift($inc_path, __DIR__ . "/../tpl/");
   $loader = new \Twig\Loader\FilesystemLoader($inc_path);
@@ -19,26 +49,27 @@ function get_twig(array $inc_path = [])
   return $twig;
 }
 
-function supported_tokens()
+function supported_tokens() : array
 {
   return [
     'POD',
     'default',
-    'optional',
+    'alias',
+    'virtual',
     'bitfields',
     'cloneable',
-    'virtual',
-    'id',
-    'owner',
-    'pkey',
     'obscured',
     'diffable',
-    'bhl_bind',
+    //TODO: this should be in a different package as a plugin
+    'table_pkey',
+    'cs_embed',
+    'cs_implements',
     'cs_attributes',
     'cs_accessor_interface',
     'cs_propget_interface',
     'cs_propset_interface',
     'cs_propgetset_interface',
+    'cs_obsolete',
     //TODO:
     //'i18n'
   ];
@@ -61,7 +92,7 @@ function _add_twig_support(\Twig\Environment $twig)
   $twig->addFunction(new \Twig\TwigFunction('has_token', 
     function($o, $token) 
     { 
-      return $o->hasToken($token);
+      return (($o instanceof \mtgMetaUnit) || ($o instanceof \mtgMetaField)) && $o->hasToken($token);
     }
   ));
   $twig->addFunction(new \Twig\TwigFunction('has_token_in_parent', 
@@ -85,12 +116,41 @@ function _add_twig_support(\Twig\Environment $twig)
         return $v;
     }
   ));
+  $twig->addFunction(new \Twig\TwigFunction('field_index', 
+    function(\mtgMetaStruct $s, string $f): int
+    {
+      $idx = get_field_index($s, $f);
+      if($idx == -1)
+      {
+        throw new Exception("field `$f` not found in `{$s->getName()}`");
+      }
+
+      return $idx;
+    }
+  ));
+  $twig->addFunction(new \Twig\TwigFunction('find_struct', 
+    function(string $name) use ($twig): \mtgMetaStruct
+    {
+      $meta = $twig->getGlobals()['meta'];
+      $unit = $meta->findUnit($name);
+      return $unit->object;
+    }
+  ));
   $twig->addFilter(new \Twig\TwigFilter('cs_type', 
     function($type) 
     {
       return cs_type($type);
     }
   ));
+  $twig->addFilter(new \Twig\TwigFilter('cs_simple_type', 
+    function($type) 
+    {
+      if($type instanceof \mtgArrType)
+        return cs_simple_type($type->getValue());
+      else
+        return cs_simple_type($type);
+    }
+  ));
   $twig->addFilter(new \Twig\TwigFilter('cs_type_prefix', 
     function($type) 
     {
@@ -136,13 +196,13 @@ function _add_twig_support(\Twig\Environment $twig)
   $twig->addFunction(new \Twig\TwigFunction('get_diff_related_units', 
     function($o) 
     { 
-      return get_diff_related_units($o);
+      return get_diff_all_related_structs($o);
     }
   ));
-  $twig->addFunction(new \Twig\TwigFunction('get_all_accessor_interfaces', 
+  $twig->addFunction(new \Twig\TwigFunction('get_all_declarable_accessor_interfaces', 
     function($o) 
     { 
-      return get_all_accessor_interfaces($o);
+      return get_all_declarable_accessor_interfaces($o);
     }
   ));
   $twig->addFunction(new \Twig\TwigFunction('get_accessor_interfaces', 
@@ -160,7 +220,7 @@ function cs_type(\mtgType $type)
   else if($type instanceof \mtgArrType)
     return "List<" . cs_simple_type($type->getValue()) . ">";
   else
-    throw new Exception("Unknown type '{$type}'");
+    throw new Exception("Unknown type '{$type->getName()}'");
 }
 
 function cs_simple_type(\mtgType $type)
@@ -196,11 +256,11 @@ function cs_simple_type(\mtgType $type)
       case "bool":
         return "bool";
       case "blob":
-        return "byte[]";
+        return "ArraySegment<byte>";
     }
     throw new Exception("Unknown type '{$type}'");
   }
-  if($type->hasToken("bhl_native_class"))
+  if($type instanceof \mtgUserType && $type->hasToken("bhl_native_class"))
     return $type->getToken("bhl_native_class");
   return $type->getName();
 }
@@ -238,7 +298,7 @@ function cs_type_prefix(\mtgType $type)
     case "blob":
       return "Blob";
     default:
-      throw new Exception("Unknown type '{$type}'");
+      throw new Exception("Unknown type '{$type->getName()}'");
   }
 }
 
@@ -307,14 +367,14 @@ function var_reset($name, \mtgType $type, $default = null)
         $str .= " = ".trim($default, '"').";";
       }
       else
-        $str .= ' = null;';
+        $str .= ' = default;';
     }
     else 
       throw new Exception("Unknown type '$type'");
   }
   else if($type instanceof \mtgArrType) 
   {
-    $str = "Meta.ClearList(ref $name);";
+    $str = "if($name == null) $name = new(); else $name.Clear();";
 
     if($default)
     {
@@ -348,72 +408,84 @@ function var_reset($name, \mtgType $type, $default = null)
   else if($type instanceof \mtgMetaStruct)
   {
     $str = "";
-    $is_pod = $type->hasToken('POD');
-    if($is_pod)
-      $str .= "$name.reset(); ";
-    else 
-      $str .= "Meta.Reset(ref $name); ";
+    if(!is_null_str($default))
+    {
+      $is_pod = $type->hasToken('POD');
+      if($is_pod)
+        $str .= "$name.Reset(); ";
+      else
+        $str .= "if($name == null) $name = new(); else $name.Reset(); ";
+    }
 
     if($default)
     {
-      $default = is_array($default) ? $default : json_decode($default, true);
-      if(!is_array($default))
-        throw new Exception("Bad default value for struct: $default");
-
-      foreach($default as $k => $v)
+      $default_val = is_array($default) ? $default : json_decode($default, true);
+      if(is_array($default_val))
       {
-        $kf = $type->getField($k); 
-        $str .= var_reset("$name." . $kf->getName(), $kf->getType(), $v);
+        foreach($default_val as $k => $v)
+        {
+          $kf = $type->getField($k); 
+          $str .= var_reset("$name." . $kf->getName(), $kf->getType(), $v);
+        }
       }
+      else if(is_null_str($default))
+        $str .= "$name = null; ";
+      else
+        throw new Exception("Bad default value for struct: " . var_export($default, true));
     }
   }
   else
-    throw new Exception("Bad type '$type'");
+    throw new Exception("Bad type '{$type->getName()}'");
   return $str;
 }
 
+function is_null_str($default)
+{
+  return is_string($default) && strtolower($default) === 'null';
+}
+
 function var_sync($fname, \mtgType $type, $buf, array $tokens, $opts)
 {
-  $optional = array_key_exists('optional', $tokens);
-  if($optional)
-    $opts .= " | MetaSyncFieldOpts.SKIP_OPTIONAL";
+  $key_name = array_key_exists('alias', $tokens) ? $tokens['alias'] : $fname; 
 
   $str = '';
   if($type instanceof \mtgBuiltinType)
   {
-    $str .= "Meta.Sync({$buf}, ref {$fname}, {$opts});\n";
+    $str .= "MetaIO.Sync({$buf}, ref {$fname}, \"{$key_name}\", {$opts});\n";
   }
   else if($type instanceof \mtgMetaStruct)
   {
     if(array_key_exists('virtual', $tokens))
-      $str .= "{$fname} = ({$type->getName()})Meta.SyncGeneric({$buf}, {$fname}, {$opts});\n";
+      $str .= "{$fname} = MetaIO.SyncGeneric({$buf}, {$fname}, \"{$key_name}\", {$opts});\n";
     else
-      $str .= "Meta.Sync({$buf}, ref {$fname}, {$opts});\n";
+      $str .= "MetaIO.Sync({$buf}, ref {$fname}, \"{$key_name}\", {$opts});\n";
   }
   else if($type instanceof \mtgMetaEnum)
   {
     $str .= "int __tmp_{$fname} = (int)$fname;\n";
-    $str .= "Meta.Sync({$buf}, ref __tmp_{$fname}, {$opts});\n";
-    $str .= "if($buf.is_read) {$fname} = ({$type->getName()})__tmp_{$fname};\n";
+    $str .= "MetaIO.Sync({$buf}, ref __tmp_{$fname}, \"{$key_name}\", {$opts});\n";
+    $str .= "if($buf.is_read) {$fname} = (".cs_type($type).")__tmp_{$fname};\n";
   }
   else if($type instanceof \mtgArrType)
   {
     if(array_key_exists('virtual', $tokens))
-      $str .= "Meta.SyncGeneric({$buf}, {$fname}, {$opts});\n";
+      $str .= "MetaIO.SyncGeneric({$buf}, {$fname}, \"{$key_name}\", {$opts});\n";
     else
     {
       if($type->getValue() instanceof \mtgMetaEnum)
       {
-        $str .= "Meta.tmp_enums_list.Clear(); if(!{$buf}.is_read) { foreach(var _enum_tmp in {$fname}) Meta.tmp_enums_list.Add((int)_enum_tmp); }\n";
-        $str .= "Meta.Sync({$buf}, Meta.tmp_enums_list, {$opts});\n";
-        $str .= "if({$buf}.is_read) foreach(var _int_tmp in Meta.tmp_enums_list) { {$fname}.Add(({$type->getValue()->getName()}) _int_tmp); }\n";
+        $str .= "{\n";
+        $str .= "var tmp_enums_list = new List<int>(); if(!{$buf}.is_read) { foreach(var _enum_tmp in {$fname}) tmp_enums_list.Add((int)_enum_tmp); }\n";
+        $str .= "MetaIO.Sync({$buf}, tmp_enums_list, \"{$key_name}\", {$opts});\n";
+        $str .= "if({$buf}.is_read) { {$fname}.Clear(); foreach(var _int_tmp in tmp_enums_list) {$fname}.Add(({$type->getValue()->getName()}) _int_tmp); }\n";
+        $str .= "}\n";
       }
       else
-        $str .= "Meta.Sync({$buf}, {$fname}, {$opts});\n";
+        $str .= "MetaIO.Sync({$buf}, {$fname}, \"{$key_name}\", {$opts});\n";
     }
   }
   else
-    throw new Exception("Unknown type '$type'");
+    throw new Exception("Unknown type '{$type->getName()}'");
 
   return $str;
 }
@@ -463,17 +535,8 @@ function fields_count_self(\mtgMetaStruct $struct)
 function is_primary_field(\mtgMetaStruct $struct, $fld)
 {
   $primary_fields = array();  
-  if($struct->hasToken("id"))
-    $primary_fields[] = $struct->getToken("id");
-
-  if($struct->hasToken("owner"))
-    $primary_fields[] = $struct->getToken("owner");
-
-  if($struct->hasToken("pkey"))
-  {
-    foreach(explode(",", $struct->getToken("pkey")) as $name)
-      $primary_fields[] = $name;
-  }
+  if($struct->hasToken("table_pkey"))
+    $primary_fields = explode(",", $struct->getToken("table_pkey"));
 
   return in_array($fld->getName(), $primary_fields);
 }
@@ -500,6 +563,36 @@ function get_diff_related_units(\mtgMetaStruct $struct)
   return $result;
 }
 
+function get_diff_all_related_structs(\mtgMetaStruct $struct): array
+{
+  $result = array_reduce($struct->getFields(), function(array $structs, \mtgMetaField $field) {
+    if($field->hasToken('nodiff'))
+    {
+      return $structs;
+    }
+
+    $type = $field->getType();
+    if($type instanceof \mtgArrType)
+    {
+      $type = $type->getValue();
+    }
+
+    if($type instanceof \mtgMetaStruct)
+    {
+      $structs[] = $type;
+    }
+
+    return $structs;
+  }, []);
+
+  foreach($result as $s)
+  {
+    $result = array_merge(get_diff_all_related_structs($s), $result);
+  }
+
+  return $result;
+}
+
 class AccessorInterfaceField
 {
   public $field;
@@ -538,9 +631,26 @@ class AccessorInterface
   }
 }
 
+//NOTE: by default, accessor interface declarations will be generated in the autogen bundle.
+//If you want your metagen struct to implement some existing C# inteface
+//use quotes and prefix the fully-qualified interface name with "!", e.g. like this:
+//
+// ticket : string @cs_accessor_interface:"!BitGames.ServerIntegration.IRpcWithAuth"
+//
 function get_accessor_interfaces(\mtgMetaStruct $struct)
+{
+  return get_accessor_interfaces_ex($struct, /* declarable_only = */ false);
+}
+
+function get_accessor_interfaces_declarable(\mtgMetaStruct $struct)
+{
+  return get_accessor_interfaces_ex($struct, /* declarable_only = */ true); 
+}
+
+function get_accessor_interfaces_ex(\mtgMetaStruct $struct, $declarable_only)
 {
   $ifs = array();
+  $external_mark = '"!';
   foreach($struct->getFields() as $field)
   {
     $is_accessor = $field->hasToken('cs_accessor_interface');
@@ -560,6 +670,14 @@ function get_accessor_interfaces(\mtgMetaStruct $struct)
       elseif($is_propgetset)
         $ai->cs_interface_name = $field->getToken('cs_propgetset_interface');
 
+      $is_external = strpos($ai->cs_interface_name, $external_mark) === 0;
+      if($is_external)
+      {
+         $ai->cs_interface_name = trim($ai->cs_interface_name, $external_mark);
+         if($declarable_only)
+           continue;
+      }
+
       if(array_key_exists($ai->getUID(), $ifs))
         $ai = $ifs[$ai->getUID()];
 
@@ -580,19 +698,70 @@ function get_accessor_interfaces(\mtgMetaStruct $struct)
   return $ifs;
 }
 
-function get_all_accessor_interfaces(\mtgMetaInfo $info)
+function get_all_declarable_accessor_interfaces(\mtgMetaInfo $info)
 {
   $all = array();
   foreach($info->getUnits() as $unit)
   {
     if($unit->object instanceof \mtgMetaStruct)
-      $all = array_merge($all, get_accessor_interfaces($unit->object));
+      $all = array_merge($all, get_accessor_interfaces_declarable($unit->object));
     else if($unit->object instanceof \mtgMetaRPC)
     {
-      $all = array_merge($all, get_accessor_interfaces($unit->object->getReq()));
-      $all = array_merge($all, get_accessor_interfaces($unit->object->getRsp()));
+      $all = array_merge($all, get_accessor_interfaces_declarable($unit->object->getReq()));
+      $all = array_merge($all, get_accessor_interfaces_declarable($unit->object->getRsp()));
     }
   }
   return $all;
 }
 
+function get_field_index(\mtgMetaStruct $struct, string $field_name): int
+{
+  $idx = -1;
+  foreach($struct->getFields() as $field)
+  {
+    $idx++;
+    if($field->getName() == $field_name)
+    {
+      return $idx;
+    }
+  }
+  return -1;
+}
+
+function paginate($total, $step)
+{
+  //pages are returned as an array where each element is in interval [N, Y)
+  $pages = array();
+
+  $steps = (int)($total/$step);
+  $rest = $total % $step;
+
+  for($i=1;$i<=$steps;++$i)
+    $pages[] = array(($i-1)*$step, $i*$step);
+
+  if($rest != 0)
+    $pages[]  = array(($i-1)*$step, ($i-1)*$step + $rest);
+
+  return $pages;
+}
+
+//slices array like this: [[idx0,[..]], [idx1,[..]], ...]
+function slice_units(array $units, $max)
+{
+  $pages = paginate(sizeof($units), $max);
+  
+  $sliced = array();
+
+  $units_keys = array_keys($units);
+
+  foreach($pages as $idx => $page)
+  {
+    $slice = array();
+    for($i = $page[0];$i<$page[1];++$i)
+    {
+      $slice[] = $units[$units_keys[$i]];
+    }
+    $sliced[$idx] = $slice;  
+  }
+  return $sliced;
+}

tpl/codegen_factory.twig

@@ -1,6 +1,7 @@
 //THIS FILE IS GENERATED AUTOMATICALLY, DO NOT TOUCH IT!
 using System.Collections.Generic;
 using metagen;
+using System;
 
 {%- import "macro.twig" as macro -%}
 
@@ -8,7 +9,9 @@ using metagen;
 namespace {{namespace}} {
 {% endif %}
 
-{{ macro.decl_units(meta) }}
+{%- for ai in get_all_declarable_accessor_interfaces(meta) ~%}
+  {{ macro.decl_accessor_interface(ai) }}
+{%- endfor ~%}
 
 static public class AutogenBundle 
 {

tpl/codegen_types_slice.twig

@@ -0,0 +1,16 @@
+//THIS FILE IS GENERATED AUTOMATICALLY, DO NOT TOUCH IT!
+using System.Collections.Generic;
+using metagen;
+using System;
+
+{%- import "macro.twig" as macro -%}
+
+{% if namespace is defined ~%}
+namespace {{namespace}} {
+{% endif %}
+
+{{ macro.decl_units(slice_units) }}
+
+{% if namespace is defined ~%}
+} //namespace {{namespace}}
+{% endif %}

tpl/macro.twig

@@ -1,7 +1,6 @@
+{% macro decl_units(units) %}
 
-{% macro decl_units(meta) %}
-
-{%- for u in meta.getunits ~%}
+{%- for u in units ~%}
   {%- if u.object is instanceof('\\mtgMetaStruct') -%}
   {{ _self.decl_struct(u.object) }}
   {%- elseif u.object is instanceof('\\mtgMetaEnum') -%}
@@ -11,13 +10,13 @@
   {%- endif ~%}
 {%- endfor ~%}
 
-{%- for ai in get_all_accessor_interfaces(meta) ~%}
-  {{ _self.decl_accessor_interface(ai) }}
-{%- endfor ~%}
-
 {% endmacro %}
 
 {% macro decl_struct(o, extra = '') %}
+
+{% set pkey_fields = token(o, 'table_pkey')|split(',') %}
+{% set pkey = pkey_fields|filter(f => f|length > 0)|map(f => o.getField(f)) %}
+
   {%- if o.parent and has_token(o, 'POD') -%}
   {{Error("@POD structs can't have a parent: " ~ o.name)}}
   {%- endif -%}
@@ -28,6 +27,8 @@
 {{_self.attributes(o)}}
 public {{_self.struct_type(o)}} {{o.name}} {{_self.base_struct_class(o)}} 
 {
+  {{token(o, 'cs_embed', '')}}
+
   {{_self.decl_struct_fields(o)}}
 
   public {{o.parent ? 'new'}} const uint STATIC_CLASS_ID = {{o.classid}};
@@ -40,58 +41,45 @@ public {{_self.struct_type(o)}} {{o.name}} {{_self.base_struct_class(o)}}
   {{_self.comment_POD_begin(o)}}
   public {{o.name}}()
   {
-    reset();
+    Reset();
   }
   {{_self.comment_POD_end(o)}}
 
-  public {{_self.override(o)}} void reset() 
+  public {{_self.override(o)}} void Reset() 
   {
     {{_self.struct_fields_reset(o)}}
   }
 
   {{_self.comment_non_cloneable_begin(o)}}
-  public {{_self.virtual_clone(o)}} void copy(IMetaStruct other)
+  public void CloneTo(ref {{o.name}} dst)
   {
-    copyFrom(({{o.name}})other);
+    MetaIO.Clone(this, ref dst, AutogenBundle.createById);
   }
 
-  public void copyFrom({{o.name}} other)
+  public {{_self.virtual_clone(o)}} IMetaStruct Clone()
   {
-    var ctx = Meta.PrepareForClone(ref other);
-    ctx.factory = AutogenBundle.createById;
-    ctx.reader.BeginContainer();
-    reset();
-    syncFields(ctx);
-    ctx.reader.EndContainer();
-{%- if has_token(o, 'bitfields') ~%} 
-    fields_mask = other.fields_mask;
-{%- endif ~%}
-  }
-
-  public {{_self.virtual_clone(o)}} IMetaStruct clone()
-  {
-    var copy = new {{o.name}}();
-    copy.copy(this);
-    return copy;
+    var dst = new {{o.name}}();
+    CloneTo(ref dst);
+    return dst;
   }
   {{_self.comment_non_cloneable_end(o)}}
 
-  public {{_self.override(o)}} void syncFields(MetaSyncContext ctx) 
+  public {{_self.override(o)}} void SyncFields(MetaSyncContext ctx) 
   {
     {{_self.sync_fields(o)}}
   }
   
-  public {{_self.override(o)}} int getFieldsCount() 
+  public {{_self.override(o)}} int GetFieldsCount() 
   {
     return {{fields_count(o)}};
   }
   
-  public {{_self.override(o)}} int getWritableFieldsCount() 
+  public {{_self.override(o)}} int GetDirtyFieldsCount() 
   {
 {%~ if has_token(o, 'bitfields') ~%}
-    return fields_mask.GetDirtyFieldsCount() + Meta.MASK_HEADER_FIELDS_COUNT;
+    return fields_mask.GetDirtyFieldsCount();
 {% else %}
-    return {{fields_count(o)}};
+    return GetFieldsCount();
 {%- endif ~%}
   }
 
@@ -99,10 +87,24 @@ public {{_self.struct_type(o)}} {{o.name}} {{_self.base_struct_class(o)}}
   {{_self.bitmask_helpers(o)}}
 {%- endif -%}
   
-{%- if has_token(o, 'id') ~%}
-  public ulong getPrimaryId() 
+{%- if has_token(o, 'table') ~%}
+  public static bool operator <({{o.name}} o1, {{o.name}} o2)
   {
-    return (ulong){{token(o, 'id')}};
+    return {{o.name}}Comparer.self.Compare(o1, o2) < 0;
+  }
+
+  public static bool operator >({{o.name}} o1, {{o.name}} o2)
+  {
+    return {{o.name}}Comparer.self.Compare(o1, o2) > 0;
+  }
+
+  public {{o.name}}Id MakeId()
+  {
+    return new {{o.name}}Id{
+      {% for f in pkey %}
+      {{f.name}} = {{f.name}},
+      {% endfor %}
+    };
   }
 {%- endif -%}
 
@@ -115,6 +117,32 @@ public {{_self.struct_type(o)}} {{o.name}} {{_self.base_struct_class(o)}}
   {{extra}}
 }
 
+{%- if has_token(o, 'table') ~%}
+public class {{o.name}}Comparer : IComparer<{{o.name}}>
+{
+  public readonly static {{o.name}}Comparer self = new {{o.name}}Comparer();
+
+  public int Compare({{o.name}} o1, {{o.name}} o2)
+  {
+    int result;
+
+    {% set comparable = pkey %}
+    {% if comparable|length > 1 %}
+      {% set comparable = comparable [1:] %}
+    {% endif %}
+
+    {% for f in comparable %}
+      {% if not loop.first %}
+      if(result == 0)
+      {% endif %}
+      result = o1.{{f.name}}.CompareTo(o2.{{f.name}});
+    {% endfor %}
+
+    return result;
+  }
+}
+{%- endif ~%}
+
 {% endmacro %}
 
 {%- macro decl_struct_fields(o) -%}
@@ -128,23 +156,43 @@ public FieldsMask fields_mask;
 
 {%- macro decl_struct_field(o, f) -%}
 {{_self.attributes(f)}}
-public {{f.type|cs_type|obscure_type(f)}} {{f.name}} {% if not has_token(o, 'POD') -%} {{_self.decl_init_value(f.type)}} {%- endif -%};
+{% if has_token(f, 'flt_i18n') -%}
+{{f.type}} _{{f.name}} = "";
+List<{{f.type}}> __{{f.name}} = new List<{{f.type}}>();
+{%- endif ~%}
+public {{f.type|cs_type|obscure_type(f)}} {{f.name}} {% if not has_token(o, 'POD') -%} {{_self.decl_init_value(f)}} {%- endif -%} {% if not has_token(f, 'flt_i18n') -%};{%- endif -%}
 {%- endmacro -%}
 
-{% macro decl_init_value(type) %}
-{%- if type is instanceof('\\mtgBuiltinType') -%}
-{%- if type.isstring -%} = ""{%- endif -%}
+{% macro decl_init_value(f) %}
+{%- if f.type is instanceof('\\mtgBuiltinType') -%}
+{%- if f.type.isstring -%}
+{% if has_token(f, 'flt_i18n') -%}
+{get{return plural_{{f.name}}(double.NaN);}set{_{{f.name}}=value;}}
+{{f.type}} plural_{{f.name}}(double force_n){return MetaI18N.I18NPick(_{{f.name}},__{{f.name}},"#",force_n);}
 {%- else -%}
-= new {{type|cs_type}}()
+= ""
+{%- endif ~%}
+{%- endif -%}
+{%- else -%}
+{%- if has_token(f, 'default') and token(f, 'default') == 'null'  -%}
+/*null*/
+{%- else -%}
+= new {{f.type|cs_type}}()
+{%- endif -%}
 {%- endif -%}
 {% endmacro %}
 
 {% macro struct_fields_reset(o) %}
 {%- if o.parent ~%} 
-base.reset();
+base.Reset();
 {%- endif -%}
 {%- for f in o.fields ~%}
-{{var_reset(f.name, f.type, token_or(f, 'default', null))}}
+{% set fname = f.name %}
+{% if has_token(f, 'flt_i18n') -%}
+{% set fname = '_' ~ f.name %}
+if(_{{fname}} == null) _{{fname}} = new(); else _{{fname}}.Clear();
+{%- endif ~%}
+{{var_reset(fname, f.type, token_or(f, 'default', null))}}
 {%- endfor -%}
 {%- if has_token(o, 'bitfields') ~%} 
 ResetFieldMask();
@@ -152,6 +200,12 @@ ResetFieldMask();
 {% endmacro %}
 
 {%- macro attributes(o) %}
+{% if has_token(o, "cs_obsolete") %}
+/// <summary>
+/// {{token(o, "cs_obsolete")}}
+/// </summary>
+[Obsolete]
+{% endif %}
 {%- if has_token(o, 'cs_attributes') -%} 
 {%- for attr in token(o, 'cs_attributes')|split(',') -%} 
 [{{attr}}]
@@ -170,6 +224,12 @@ ResetFieldMask();
 : {{o.parent ? o.parent.name : 'BaseMetaStruct'}}
 {{- has_token(o, 'cloneable') and not o.parent ? ', IMetaCloneable' -}}
 {%- endif -%}
+{%- if has_token(o, "cs_implements") -%}
+{%- for ifs in token(o, "cs_implements")|split(',') -%} 
+, {{ifs}}
+{%- endfor -%} 
+{%- endif -%} 
+
 {%- for ai in get_accessor_interfaces(o) -%} 
 , {{ai.interfacename}}
 {%- endfor -%} 
@@ -224,14 +284,18 @@ override
 
 {%- macro sync_fields(o) -%}
 {%- if has_token(o, 'bitfields') ~%}
-var bitctx = new Meta.BitfieldsContext(ctx, fields_mask);
+var bitctx = new MetaIO.BitfieldsContext(ctx, fields_mask);
 bitctx.SyncMaskHeader();
 {%- endif -%}
 {%- if o.parent ~%} 
-base.syncFields(ctx);
+base.SyncFields(ctx);
 {%- endif -%}
 {%- for f in o.fields ~%}
-{{var_sync(f.name, f.type, 'ctx', f.tokens, get_sync_opts(o, 'bitctx'))}}
+{% set fname = f.name %}
+{% if has_token(f, 'flt_i18n') -%}
+{% set fname = '_' ~ f.name %}
+{%- endif -%}
+{{var_sync(fname, f.type, 'ctx', f.tokens, get_sync_opts(o, 'bitctx'))}}
 {%- endfor -%}
 {%- endmacro -%}
 
@@ -243,14 +307,34 @@ base.syncFields(ctx);
   }
 
   public void SetPrimaryFieldsChanged()
+  {
+    SetPrimaryFieldsChanged(ref this.fields_mask);
+  }
+
+  static void SetPrimaryFieldsChanged(ref FieldsMask mask)
   {
 {%- for f in o.fields ~%}
 {%- if is_primary_field(o, f) ~%}
-    Meta.SetFieldDirty(ref fields_mask, {{loop.index0}});
+    MetaIO.SetFieldDirty(ref mask, {{loop.index0}});
 {%- endif -%}
 {%- endfor ~%}
   }
 
+  public bool IsContentDirty()
+  {
+    int fields_amount = {{fields_count(o)}};
+    var primary_id_mask = FieldsMask.MakeClean(fields_amount); 
+    SetPrimaryFieldsChanged(ref primary_id_mask);
+    for(int i = 0; i < fields_amount; i++)
+    {
+      bool is_primary_id = primary_id_mask.IsDirty(i);
+      if(!is_primary_id && fields_mask.IsDirty(i))
+        return true;
+    }
+    
+    return false;
+  }
+
   public void SetDirtyMask()
   {
     fields_mask = FieldsMask.MakeDirty(fields_amount: {{fields_count(o)}});
@@ -287,38 +371,45 @@ public enum {{o.name}}
 }
 {% endmacro %}
 
-{% macro decl_rpc(o) %}
+{% macro decl_rpc(o, is_global = true) %}
 
+{% if is_global %}
 {{_self.decl_struct(o.req)}}
 {{_self.decl_struct(o.rsp)}}
+{% endif %}
 
 public class {{o.name}} : IRpc
 {
+{% if not is_global %}
+  {{_self.decl_struct(o.req)}}
+  {{_self.decl_struct(o.rsp)}}
+{% endif %}
+
   public IRpcError error = null;
   public {{o.req.name}} req = new {{o.req.name}}();
   public {{o.rsp.name}} rsp = new {{o.rsp.name}}();
 
-  public int getCode() 
+  public int GetCode() 
   {
     return {{o.code}};
   }
 
-  public void setError(IRpcError error)
+  public void SetError(IRpcError error)
   {
     this.error = error;
   }
 
-  public IRpcError getError()
+  public IRpcError GetError()
   {
     return error;
   }
 
-  public IMetaStruct getRequest()
+  public IMetaStruct GetRequest()
   {
     return req as IMetaStruct;
   }
 
-  public IMetaStruct getResponse()
+  public IMetaStruct GetResponse()
   {
     return rsp as IMetaStruct;
   }
@@ -328,10 +419,12 @@ public class {{o.name}} : IRpc
 
 {%- macro diffable_support(o) -%}
 
-    public bool GetDiff({{o.name}} old, {{o.name}} diff = null, List<DataClassIds> removed = null)
+    public bool GetDiff({{o.name}} old, {{o.name}} diff = null, {{o.name}}RemovedIds removed = null)
     {
       if(diff != null) 
-        diff.reset();
+        diff.Reset();
+      if(removed != null)
+        removed.Reset();
       bool no_changes = true;
 
     {% set field_idx = -1 %}
@@ -339,63 +432,38 @@ public class {{o.name}} : IRpc
       {% set field_idx = field_idx + 1 %}
       
       {%- if not has_token(f, 'nodiff') -%}
-      {{_self.field_diff(f, field_idx)}}
+      {{_self.field_diff(f, field_idx, o)}}
       {%- endif -%}
     {% endfor %}
 
      return !no_changes;
     }
     
+    {% set unique_diff_units = [] %}
     {% for u in get_diff_related_units(o) %}
-    {{_self.diff_methods_for(u)}}
+      {% if u not in unique_diff_units %}
+        {{_self.diff_methods_for(u)}}
+        {% set unique_diff_units = unique_diff_units|merge([u]) %}
+      {% endif %}
     {% endfor %}
 
-    static public List<ulong> GetClassRemovedIds(List<DataClassIds> removed_ids, uint class_id) 
-    {
-      if(removed_ids == null)
-        return null;
-      DataClassIds res;
-      if(!FindClassRemovedIds(removed_ids, class_id, out res))
-      {
-        res = new DataClassIds();
-        res.class_id = class_id;
-        res.ids = new List<ulong>();
-        removed_ids.Add(res);
-      }
-      return res.ids;
-    }
-
-    static bool FindClassRemovedIds(List<DataClassIds> removed_ids, uint class_id, out DataClassIds res)
-    {
-      for(int i=0;i<removed_ids.Count;++i)
-      {
-        if(removed_ids[i].class_id == class_id)
-        {
-          res = removed_ids[i];
-          return true;
-        }
-      }
-      res = default(DataClassIds);
-      return false;
-    }
-
 {% endmacro %}
 
 {%- macro diff_methods_for(u) -%}
-
+  {% if has_token(u, 'table') %}
   static public bool DiffOne(ref {{u.name}} curr, {{u.name}} old)
   {
     return !{{_self.compare_func_name(u)}}(ref curr, old);
   }
 
-  static public bool DiffCollection(List<{{u.name}}> items, List<{{u.name}}> old_items, List<{{u.name}}> changed, List<DataClassIds> removed)
+  static public bool DiffCollection(List<{{u.name}}> items, List<{{u.name}}> old_items, List<{{u.name}}> changed, List<{{u.name}}Id> removed)
   {
     bool has_diff = false;
     if(changed != null)
       changed.Clear();
 
-    SortByPrimaryId(ref items);
-    SortByPrimaryId(ref old_items);
+    items.Sort({{u.name}}Comparer.self);
+    old_items.Sort({{u.name}}Comparer.self);
 
     int i = 0;
     int old_i = 0;
@@ -404,17 +472,16 @@ public class {{o.name}} : IRpc
       var old = old_items[old_i];
       var item = items[i];
 
-      if(old.getPrimaryId() < item.getPrimaryId())
+      if(old < item)
       {
-        var removed_ids = GetClassRemovedIds(removed, {{u.name}}.STATIC_CLASS_ID);
-        if(removed_ids != null)
-          removed_ids.Add(old.getPrimaryId());
+        if(removed != null)
+          removed.Add(old.MakeId());
         old_i++;
         has_diff = true;
       }
-      else if(old.getPrimaryId() > item.getPrimaryId())
+      else if(old > item)
       {
-        item.SetDirtyMask();
+        item.SetDirtyMaskDeep();
         if(changed != null)
           changed.Add(item);
         i++;
@@ -422,7 +489,7 @@ public class {{o.name}} : IRpc
       }
       else
       {
-        if(!{{_self.compare_func_name(u)}}(ref item, old))
+        if(!{{_self.compare_func_name(u)}}(ref item, old) && item.IsContentDirty())
         {
           if(changed != null)
             changed.Add(item);
@@ -438,7 +505,7 @@ public class {{o.name}} : IRpc
     for(; i < items.Count; i++)
     {
       var item = items[i];
-      item.SetDirtyMask();
+      item.SetDirtyMaskDeep();
       if(changed != null)
         changed.Add(item);
       has_diff = true;
@@ -447,38 +514,19 @@ public class {{o.name}} : IRpc
     //old items leftovers are considered removed
     for(; old_i < old_items.Count; old_i++)
     {
-      var removed_ids = GetClassRemovedIds(removed, {{u.name}}.STATIC_CLASS_ID);
-      if(removed_ids != null)
-        removed_ids.Add(old_items[old_i].getPrimaryId());
+      if(removed != null)
+        removed.Add(old_items[old_i].MakeId());
       has_diff = true;
     }
 
     //if(changed.Count > 0)
-    //  Meta.LogDebug("Changed in collection {{u.name}}");
+    //  MetaIO.LogDebug("Changed in collection {{u.name}}");
     //if(removed_ids.Count > 0)
-    //  Meta.LogDebug("Removed in collection {{u.name}}");
+    //  MetaIO.LogDebug("Removed in collection {{u.name}}");
 
     return has_diff;
   }
-
-  class {{u.name}}Compare : IComparer<{{u.name}}>
-  {
-    public int Compare({{u.name}} o1, {{u.name}} o2)
-    { 
-      var id1 = o1.getPrimaryId();
-      var id2 = o2.getPrimaryId();
-      if(id1 == id2)
-        return 0;
-      return id1 > id2 ? 1 : -1;
-    }
-  }
-
-  static {{u.name}}Compare {{u.name}}_compare = new {{u.name}}Compare();
-
-  static void SortByPrimaryId(ref List<{{u.name}}> list)
-  {
-    list.Sort({{u.name}}_compare); 
-  }
+  {% endif %}
 
   public static bool {{_self.compare_func_name(u)}}(ref {{u.name}} a, {{u.name}} b)
   {
@@ -506,33 +554,127 @@ public class {{o.name}} : IRpc
 {{has_token(o, 'bitfields')?'CompareAndMarkFields':'IsEqual'}}
 {%- endmacro %}
 
-{%- macro field_compare(o, f, field_idx) -%}
+{% macro field_compare_array_bitfields_nested_diffable(o, f, field_idx) %}
+    {
+      int {{f.name}}_count_a = a.{{f.name}} == null ? 0 : a.{{f.name}}.Count;
+      int {{f.name}}_count_b = b.{{f.name}} == null ? 0 : b.{{f.name}}.Count;
 
-  {% if f.type is instanceof('\\mtgArrType') ~%}
+      //For NESTED arrays of DIFFABLE structs:
+      //for DB compatibility we must add the WHOLE ARRAY to the diff if ANY ELEMENT of the array has changed.
+      //That means marking each element's fields_mask as dirty
+
+      //Changed size means the array has changed, no furhter checks required.
+      bool {{f.name}}_equal = {{f.name}}_count_a == {{f.name}}_count_b;
+
+      //For same size arrays - check contents for changes.
+      if({{f.name}}_equal)
+      {
+        for(int i=0;i<{{f.name}}_count_a && i<{{f.name}}_count_b;++i)
+        {
+          var tmp_{{f.name}} = a.{{f.name}}[i];
+          if(!CompareAndMarkFields(ref tmp_{{f.name}}, b.{{f.name}}[i]))
+          {
+            {{f.name}}_equal = false;
+            break;
+          }
+        }
+      }
+
+      //If change detected - mark top-level field as changed and add the whole array to the diff
+      if(!{{f.name}}_equal)
+      {
+        MetaIO.SetFieldDirty(ref a.fields_mask, {{field_idx}});
+        for(int i=0; i<{{f.name}}_count_a;++i)
+        {
+          var tmp_{{f.name}} = a.{{f.name}}[i];
+          tmp_{{f.name}}.SetDirtyMaskDeep();
+          a.{{f.name}}[i] = tmp_{{f.name}};
+          is_equal = false;
+        }
+      }
+    }
+{%- endmacro %}
+
+{% macro field_compare_array_bitfields_nested_plain(o, f, field_idx) %}
+    if(a.{{f.name}} == null ||
+       b.{{f.name}} == null ||
+       a.{{f.name}}.Count != b.{{f.name}}.Count)
+    {
+      MetaIO.SetFieldDirty(ref a.fields_mask, {{field_idx}});
+      is_equal = false;
+    }
+    else
+    {
+      for(int i=0;i<a.{{f.name}}.Count;++i)
+      {
+        {%- if f.type.value is instanceof('\\mtgMetaStruct') ~%}
+          var tmp_{{f.name}} = a.{{f.name}}[i];
+          if(!{{_self.compare_func_name(f.type.value)}}(ref tmp_{{f.name}}, b.{{f.name}}[i]))
+          {
+            MetaIO.SetFieldDirty(ref a.fields_mask, {{field_idx}});
+            is_equal = false;
+            break;
+          }
+        {%- else -%}
+          if(a.{{f.name}}[i] != b.{{f.name}}[i])
+          {
+            MetaIO.SetFieldDirty(ref a.fields_mask, {{field_idx}});
+            is_equal = false;
+            break;
+          }
+        {%- endif -%}
+      }
+    }
+{%- endmacro %}
+
+{% macro field_compare_array_plain(o, f, field_idx) %}
     if(a.{{f.name}} == null || 
        b.{{f.name}} == null || 
        a.{{f.name}}.Count != b.{{f.name}}.Count) 
       return false;
-
     for(int i=0;i<a.{{f.name}}.Count;++i) 
     {
       {%- if f.type.value is instanceof('\\mtgMetaStruct') ~%}
-        if(!IsEqual(ref a.{{f.name}}[i], b.{{f.name}}[i])) 
+        var tmp_{{f.name}} = a.{{f.name}}[i];
+        if(!{{_self.compare_func_name(f.type.value)}}(ref tmp_{{f.name}}, b.{{f.name}}[i]))
           return false;
       {%- else -%}
-        if(a.{{f.name}}[i] != b.{{f.name}}[i]) 
+        if(a.{{f.name}}[i] != b.{{f.name}}[i])
           return false;
       {%- endif -%}
     }
+{%- endmacro %}
+
+{%- macro field_compare(o, f, field_idx) -%}
+
+  {% if f.type is instanceof('\\mtgArrType') ~%}
+    {% if has_token(o, 'bitfields') ~%}
+      {% if has_token(f.type.value, 'bitfields') ~%}
+        {{_self.field_compare_array_bitfields_nested_diffable(o, f, field_idx)}}
+      {% else ~%}
+        {{_self.field_compare_array_bitfields_nested_plain(o, f, field_idx)}}
+      {% endif ~%}
+    {% else ~%}
+      {{_self.field_compare_array_plain(o, f, field_idx)}}
+    {% endif ~%}
+
   {% elseif f.type is instanceof('\\mtgMetaStruct') ~%}
-    if(!IsEqual(ref a.{{f.name}}, b.{{f.name}})) 
+    var tmp_{{f.name}} = a.{{f.name}}[i];
+    if(!{{_self.compare_func_name(f.type.value)}}(ref tmp_{{f.name}}, b.{{f.name}}))
+    {% if has_token(o, 'bitfields') ~%}
+    {
+      MetaIO.SetFieldDirty(ref a.fields_mask, {{field_idx}});
+      is_equal = false;
+    }
+    {% else ~%}
       return false;
+    {% endif ~%}
   {% elseif f.type is instanceof('\\mtgBuiltinType') and f.type.isstring ~%}
       if((a.{{f.name}} == null ? "" : a.{{f.name}}) 
       != (b.{{f.name}} == null ? "" : b.{{f.name}}))
       {% if has_token(o, 'bitfields') ~%}
       {
-        Meta.SetFieldDirty(ref a.fields_mask, {{field_idx}});
+        MetaIO.SetFieldDirty(ref a.fields_mask, {{field_idx}});
         is_equal = false;
       }
       {% else ~%}
@@ -542,7 +684,7 @@ public class {{o.name}} : IRpc
       if(a.{{f.name}} != b.{{f.name}})
       {% if has_token(o, 'bitfields') ~%}
       {
-        Meta.SetFieldDirty(ref a.fields_mask, {{field_idx}});
+        MetaIO.SetFieldDirty(ref a.fields_mask, {{field_idx}});
         is_equal = false;
       }
       {% else ~%}
@@ -552,11 +694,11 @@ public class {{o.name}} : IRpc
 
 {% endmacro %}
 
-{%- macro field_diff(f, field_idx) -%}
+{%- macro field_diff(f, field_idx, o) -%}
 
 {% if f.type is instanceof('\\mtgMetaStruct') %}
   {%- if not has_token(f.type, 'POD') -%}
-    Error("Diffable struct '" ~ f.type ~ "' must be POD");
+  {{Error("Diffable struct '" ~ f.type ~ "' must be POD")}}
   {%- endif -%}
 
   {
@@ -567,23 +709,25 @@ public class {{o.name}} : IRpc
       if(diff != null) 
       { 
         diff.{{f.name}} = __tmp; 
-        Meta.SetFieldDirty(ref diff.fields_mask, {{field_idx}}); 
+        MetaIO.SetFieldDirty(ref diff.fields_mask, {{field_idx}}); 
       } 
     }
   }
 {% elseif f.type is instanceof('\\mtgArrType') %}
   {%- if not has_token(f.type.value, 'POD') -%}
-    Error("Diffable struct '" ~ f.type.value ~ "' must be POD");
+  {{Error("Diffable struct '" ~ f.type.value ~ "' must be POD")}}
   {%- endif -%}
 
   if(DiffCollection({{f.name}}, old.{{f.name}}, 
       diff == null ? null : diff.{{f.name}}, 
-      removed
+      removed == null ? null : removed.{{f.name}}
      )) 
   { 
     no_changes &= false; 
     if(diff != null) 
-      Meta.SetFieldDirty(ref diff.fields_mask, {{field_idx}}); 
+      MetaIO.SetFieldDirty(ref diff.fields_mask, {{field_idx}});
+    if(removed != null && removed.{{f.name}}.Count > 0)
+      MetaIO.SetFieldDirty(ref removed.fields_mask, {{ field_index(find_struct(o.name ~ 'RemovedIds'), f.name) }});
   }
 {% elseif f.type is instanceof('\\mtgBuiltinType') %}
   if(diff != null) 
@@ -593,10 +737,10 @@ public class {{o.name}} : IRpc
   { 
     no_changes &= false; 
     if(diff != null) 
-      Meta.SetFieldDirty(ref diff.fields_mask, {{field_idx}}); 
+      MetaIO.SetFieldDirty(ref diff.fields_mask, {{field_idx}}); 
   }
 {% else %}
-    Error("Diff for field '"~ f.name ~"' is not supported");
+  {{Error("Diff for field '"~ f.name ~"' is not supported") }}
 {% endif %}
 
 {% endmacro %}
@@ -651,4 +795,3 @@ public interface {{ai.interfacename}}
 {% endfor %}
 {%- endfor ~%}
 {% endmacro %}
-