Better check for null default values

.gitea/workflows/build_composer.yaml

@@ -0,0 +1,30 @@
+
+
+name: Publish PHP Package
+
+on:
+  push:
+    tags:
+      - 'v*'
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Get tag name
+      run: echo "TAG=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
+
+    - name: zip and send
+      run: |
+        ls -la
+        apt-get update -y
+        apt-get install -y zip
+        cd ../
+        zip -r ${{ gitea.event.repository.name }}.zip ${{ gitea.event.repository.name }} -x '*.git*'
+        curl -v \
+        --user composer-pbl:${{ secrets.COMPOSER_PSWD }} \
+        --upload-file ${{ gitea.event.repository.name }}.zip \
+        https://git.bit5.ru/api/packages/bit/composer?version=${{ env.TAG }}              

CHANGELOG.md

@@ -0,0 +1,24 @@
+## v6.1.0
+- Removed redundant accessor interface declarations in generated slices
+
+## v5.0.0
+- Splitting one huge codegenerated file into multiple ones
+
+## v4.10.0
+- Added @flt_i18n token to support translations
+
+## v4.9.2
+- Added @cs_obsolete token, that mark fields with [Obsolete]. @cs_obsolete:"Comment" will add summary with "Comment" to field
+
+## v4.9.1
+- Fixing nested diffable POD structs being incorrectly marked as clean in GetDiff
+
+## v4.8.2
+- Fixing thread safety possible issue when reading an array of enums 
+- Fixing bug when the target enum array property wouldn't be cleared before reading data 
+
+## v4.8.0
+- Using C# built-in initialization routines instead of MetaIO's methods
+
+## v4.7.0
+- Array comparison generated code now takes bitfields token into account for array entries 

README.md

@@ -2,12 +2,10 @@ This package is used for code generation of C# meta structs using Twig templates
 
 Usage example:
 
-    $twig = \metagen_cs\get_twig();
-    file_put_contents('bundle.cs', 
-      $twig->render("codegen_bundle.twig", 
-        [
-         'namespace' => 'BitGames.Autogen',
-         'meta' => get_meta()
-        ]
-      )
-    );
+    $output = \metagen_cs\codegen(null, get_meta(), 
+    [
+      'namespace' => 'BitGames.Autogen'
+    ]);
+    foreach($output as $name => $text)
+      file_put_contents($name, $text); 
+

src/codegen.inc.php

@@ -2,7 +2,37 @@
 namespace metagen_cs;
 use Exception;
 
-function get_twig(array $inc_path = [])
+function codegen(?string $cache_dir, \mtgMetaInfo $meta, array $options = []) : array
+{
+  $twig = get_twig();
+
+  if(!empty($cache_dir))
+    $twig->setCache($cache_dir);
+
+  $twig->addGlobal('meta', $meta);
+
+  $options['meta'] = $meta;
+  if(!isset($options['namespace']))
+    $options['namespace'] = 'BitGames.Autogen';
+
+  $sliced_units = slice_units($meta->getUnits(), 50);
+
+  $output = array();
+
+  foreach($sliced_units as $slice_idx => $slice_units)
+  {
+    $slice_options = $options;
+    $slice_options['slice_idx'] = $slice_idx;
+    $slice_options['slice_units'] = $slice_units;
+    $output['types_'.$slice_idx.'.cs'] = $twig->render('codegen_types_slice.twig', $slice_options);
+  }
+
+  $output['factory.cs'] = $twig->render('codegen_factory.twig', $options);
+
+  return $output;
+}
+
+function get_twig(array $inc_path = []) : \Twig\Environment
 {
   array_unshift($inc_path, __DIR__ . "/../tpl/");
   $loader = new \Twig\Loader\FilesystemLoader($inc_path);
@@ -19,11 +49,12 @@ function get_twig(array $inc_path = [])
   return $twig;
 }
 
-function supported_tokens()
+function supported_tokens() : array
 {
   return [
     'POD',
     'default',
+    'alias',
     'virtual',
     'bitfields',
     'cloneable',
@@ -38,6 +69,7 @@ function supported_tokens()
     'cs_propget_interface',
     'cs_propset_interface',
     'cs_propgetset_interface',
+    'cs_obsolete',
     //TODO:
     //'i18n'
   ];
@@ -60,7 +92,7 @@ function _add_twig_support(\Twig\Environment $twig)
   $twig->addFunction(new \Twig\TwigFunction('has_token', 
     function($o, $token) 
     { 
-      return $o->hasToken($token);
+      return (($o instanceof \mtgMetaUnit) || ($o instanceof \mtgMetaField)) && $o->hasToken($token);
     }
   ));
   $twig->addFunction(new \Twig\TwigFunction('has_token_in_parent', 
@@ -110,6 +142,15 @@ function _add_twig_support(\Twig\Environment $twig)
       return cs_type($type);
     }
   ));
+  $twig->addFilter(new \Twig\TwigFilter('cs_simple_type', 
+    function($type) 
+    {
+      if($type instanceof \mtgArrType)
+        return cs_simple_type($type->getValue());
+      else
+        return cs_simple_type($type);
+    }
+  ));
   $twig->addFilter(new \Twig\TwigFilter('cs_type_prefix', 
     function($type) 
     {
@@ -155,7 +196,7 @@ function _add_twig_support(\Twig\Environment $twig)
   $twig->addFunction(new \Twig\TwigFunction('get_diff_related_units', 
     function($o) 
     { 
-      return get_diff_related_units($o);
+      return get_diff_all_related_structs($o);
     }
   ));
   $twig->addFunction(new \Twig\TwigFunction('get_all_declarable_accessor_interfaces', 
@@ -215,7 +256,7 @@ function cs_simple_type(\mtgType $type)
       case "bool":
         return "bool";
       case "blob":
-        return "byte[]";
+        return "ArraySegment<byte>";
     }
     throw new Exception("Unknown type '{$type}'");
   }
@@ -326,14 +367,14 @@ function var_reset($name, \mtgType $type, $default = null)
         $str .= " = ".trim($default, '"').";";
       }
       else
-        $str .= ' = null;';
+        $str .= ' = default;';
     }
     else 
       throw new Exception("Unknown type '$type'");
   }
   else if($type instanceof \mtgArrType) 
   {
-    $str = "MetaIO.ClearList(ref $name);";
+    $str = "if($name == null) $name = new(); else $name.Clear();";
 
     if($default)
     {
@@ -373,21 +414,21 @@ function var_reset($name, \mtgType $type, $default = null)
       if($is_pod)
         $str .= "$name.Reset(); ";
       else
-        $str .= "MetaIO.Reset(ref $name); ";
+        $str .= "if($name == null) $name = new(); else $name.Reset(); ";
     }
 
     if($default)
     {
-      $default = is_array($default) ? $default : json_decode($default, true);
-      if(is_array($default))
+      $default_val = is_array($default) ? $default : json_decode($default, true);
+      if(is_array($default_val))
       {
-        foreach($default as $k => $v)
+        foreach($default_val as $k => $v)
         {
           $kf = $type->getField($k); 
           $str .= var_reset("$name." . $kf->getName(), $kf->getType(), $v);
         }
       }
-      else if($default === null)
+      else if(is_null_str($default))
         $str .= "$name = null; ";
       else
         throw new Exception("Bad default value for struct: " . var_export($default, true));
@@ -400,7 +441,7 @@ function var_reset($name, \mtgType $type, $default = null)
 
 function is_null_str($default)
 {
-  return is_string($default) && json_decode($default, true) === null;
+  return is_string($default) && strtolower($default) === 'null';
 }
 
 function var_sync($fname, \mtgType $type, $buf, array $tokens, $opts)
@@ -423,7 +464,7 @@ function var_sync($fname, \mtgType $type, $buf, array $tokens, $opts)
   {
     $str .= "int __tmp_{$fname} = (int)$fname;\n";
     $str .= "MetaIO.Sync({$buf}, ref __tmp_{$fname}, \"{$key_name}\", {$opts});\n";
-    $str .= "if($buf.is_read) {$fname} = ({$type->getName()})__tmp_{$fname};\n";
+    $str .= "if($buf.is_read) {$fname} = (".cs_type($type).")__tmp_{$fname};\n";
   }
   else if($type instanceof \mtgArrType)
   {
@@ -433,9 +474,11 @@ function var_sync($fname, \mtgType $type, $buf, array $tokens, $opts)
     {
       if($type->getValue() instanceof \mtgMetaEnum)
       {
-        $str .= "MetaIO.tmp_enums_list.Clear(); if(!{$buf}.is_read) { foreach(var _enum_tmp in {$fname}) MetaIO.tmp_enums_list.Add((int)_enum_tmp); }\n";
-        $str .= "MetaIO.Sync({$buf}, MetaIO.tmp_enums_list, \"{$key_name}\", {$opts});\n";
-        $str .= "if({$buf}.is_read) foreach(var _int_tmp in MetaIO.tmp_enums_list) { {$fname}.Add(({$type->getValue()->getName()}) _int_tmp); }\n";
+        $str .= "{\n";
+        $str .= "var tmp_enums_list = new List<int>(); if(!{$buf}.is_read) { foreach(var _enum_tmp in {$fname}) tmp_enums_list.Add((int)_enum_tmp); }\n";
+        $str .= "MetaIO.Sync({$buf}, tmp_enums_list, \"{$key_name}\", {$opts});\n";
+        $str .= "if({$buf}.is_read) { {$fname}.Clear(); foreach(var _int_tmp in tmp_enums_list) {$fname}.Add(({$type->getValue()->getName()}) _int_tmp); }\n";
+        $str .= "}\n";
       }
       else
         $str .= "MetaIO.Sync({$buf}, {$fname}, \"{$key_name}\", {$opts});\n";
@@ -520,6 +563,36 @@ function get_diff_related_units(\mtgMetaStruct $struct)
   return $result;
 }
 
+function get_diff_all_related_structs(\mtgMetaStruct $struct): array
+{
+  $result = array_reduce($struct->getFields(), function(array $structs, \mtgMetaField $field) {
+    if($field->hasToken('nodiff'))
+    {
+      return $structs;
+    }
+
+    $type = $field->getType();
+    if($type instanceof \mtgArrType)
+    {
+      $type = $type->getValue();
+    }
+
+    if($type instanceof \mtgMetaStruct)
+    {
+      $structs[] = $type;
+    }
+
+    return $structs;
+  }, []);
+
+  foreach($result as $s)
+  {
+    $result = array_merge(get_diff_all_related_structs($s), $result);
+  }
+
+  return $result;
+}
+
 class AccessorInterfaceField
 {
   public $field;
@@ -654,3 +727,41 @@ function get_field_index(\mtgMetaStruct $struct, string $field_name): int
   }
   return -1;
 }
+
+function paginate($total, $step)
+{
+  //pages are returned as an array where each element is in interval [N, Y)
+  $pages = array();
+
+  $steps = (int)($total/$step);
+  $rest = $total % $step;
+
+  for($i=1;$i<=$steps;++$i)
+    $pages[] = array(($i-1)*$step, $i*$step);
+
+  if($rest != 0)
+    $pages[]  = array(($i-1)*$step, ($i-1)*$step + $rest);
+
+  return $pages;
+}
+
+//slices array like this: [[idx0,[..]], [idx1,[..]], ...]
+function slice_units(array $units, $max)
+{
+  $pages = paginate(sizeof($units), $max);
+  
+  $sliced = array();
+
+  $units_keys = array_keys($units);
+
+  foreach($pages as $idx => $page)
+  {
+    $slice = array();
+    for($i = $page[0];$i<$page[1];++$i)
+    {
+      $slice[] = $units[$units_keys[$i]];
+    }
+    $sliced[$idx] = $slice;  
+  }
+  return $sliced;
+}

tpl/codegen_factory.twig

@@ -1,6 +1,7 @@
 //THIS FILE IS GENERATED AUTOMATICALLY, DO NOT TOUCH IT!
 using System.Collections.Generic;
 using metagen;
+using System;
 
 {%- import "macro.twig" as macro -%}
 
@@ -8,7 +9,9 @@ using metagen;
 namespace {{namespace}} {
 {% endif %}
 
-{{ macro.decl_units(meta) }}
+{%- for ai in get_all_declarable_accessor_interfaces(meta) ~%}
+  {{ macro.decl_accessor_interface(ai) }}
+{%- endfor ~%}
 
 static public class AutogenBundle 
 {

tpl/codegen_types_slice.twig

@@ -0,0 +1,16 @@
+//THIS FILE IS GENERATED AUTOMATICALLY, DO NOT TOUCH IT!
+using System.Collections.Generic;
+using metagen;
+using System;
+
+{%- import "macro.twig" as macro -%}
+
+{% if namespace is defined ~%}
+namespace {{namespace}} {
+{% endif %}
+
+{{ macro.decl_units(slice_units) }}
+
+{% if namespace is defined ~%}
+} //namespace {{namespace}}
+{% endif %}

tpl/macro.twig

@@ -1,6 +1,6 @@
-{% macro decl_units(meta) %}
+{% macro decl_units(units) %}
 
-{%- for u in meta.getunits ~%}
+{%- for u in units ~%}
   {%- if u.object is instanceof('\\mtgMetaStruct') -%}
   {{ _self.decl_struct(u.object) }}
   {%- elseif u.object is instanceof('\\mtgMetaEnum') -%}
@@ -10,10 +10,6 @@
   {%- endif ~%}
 {%- endfor ~%}
 
-{%- for ai in get_all_declarable_accessor_interfaces(meta) ~%}
-  {{ _self.decl_accessor_interface(ai) }}
-{%- endfor ~%}
-
 {% endmacro %}
 
 {% macro decl_struct(o, extra = '') %}
@@ -160,12 +156,23 @@ public FieldsMask fields_mask;
 
 {%- macro decl_struct_field(o, f) -%}
 {{_self.attributes(f)}}
-public {{f.type|cs_type|obscure_type(f)}} {{f.name}} {% if not has_token(o, 'POD') -%} {{_self.decl_init_value(f)}} {%- endif -%};
+{% if has_token(f, 'flt_i18n') -%}
+{{f.type}} _{{f.name}} = "";
+List<{{f.type}}> __{{f.name}} = new List<{{f.type}}>();
+{%- endif ~%}
+public {{f.type|cs_type|obscure_type(f)}} {{f.name}} {% if not has_token(o, 'POD') -%} {{_self.decl_init_value(f)}} {%- endif -%} {% if not has_token(f, 'flt_i18n') -%};{%- endif -%}
 {%- endmacro -%}
 
 {% macro decl_init_value(f) %}
 {%- if f.type is instanceof('\\mtgBuiltinType') -%}
-{%- if f.type.isstring -%} = ""{%- endif -%}
+{%- if f.type.isstring -%}
+{% if has_token(f, 'flt_i18n') -%}
+{get{return plural_{{f.name}}(double.NaN);}set{_{{f.name}}=value;}}
+{{f.type}} plural_{{f.name}}(double force_n){return MetaI18N.I18NPick(_{{f.name}},__{{f.name}},"#",force_n);}
+{%- else -%}
+= ""
+{%- endif ~%}
+{%- endif -%}
 {%- else -%}
 {%- if has_token(f, 'default') and token(f, 'default') == 'null'  -%}
 /*null*/
@@ -180,7 +187,12 @@ public {{f.type|cs_type|obscure_type(f)}} {{f.name}} {% if not has_token(o, 'POD
 base.Reset();
 {%- endif -%}
 {%- for f in o.fields ~%}
-{{var_reset(f.name, f.type, token_or(f, 'default', null))}}
+{% set fname = f.name %}
+{% if has_token(f, 'flt_i18n') -%}
+{% set fname = '_' ~ f.name %}
+if(_{{fname}} == null) _{{fname}} = new(); else _{{fname}}.Clear();
+{%- endif ~%}
+{{var_reset(fname, f.type, token_or(f, 'default', null))}}
 {%- endfor -%}
 {%- if has_token(o, 'bitfields') ~%} 
 ResetFieldMask();
@@ -188,6 +200,12 @@ ResetFieldMask();
 {% endmacro %}
 
 {%- macro attributes(o) %}
+{% if has_token(o, "cs_obsolete") %}
+/// <summary>
+/// {{token(o, "cs_obsolete")}}
+/// </summary>
+[Obsolete]
+{% endif %}
 {%- if has_token(o, 'cs_attributes') -%} 
 {%- for attr in token(o, 'cs_attributes')|split(',') -%} 
 [{{attr}}]
@@ -273,7 +291,11 @@ bitctx.SyncMaskHeader();
 base.SyncFields(ctx);
 {%- endif -%}
 {%- for f in o.fields ~%}
-{{var_sync(f.name, f.type, 'ctx', f.tokens, get_sync_opts(o, 'bitctx'))}}
+{% set fname = f.name %}
+{% if has_token(f, 'flt_i18n') -%}
+{% set fname = '_' ~ f.name %}
+{%- endif -%}
+{{var_sync(fname, f.type, 'ctx', f.tokens, get_sync_opts(o, 'bitctx'))}}
 {%- endfor -%}
 {%- endmacro -%}
 
@@ -349,13 +371,20 @@ public enum {{o.name}}
 }
 {% endmacro %}
 
-{% macro decl_rpc(o) %}
+{% macro decl_rpc(o, is_global = true) %}
 
+{% if is_global %}
 {{_self.decl_struct(o.req)}}
 {{_self.decl_struct(o.rsp)}}
+{% endif %}
 
 public class {{o.name}} : IRpc
 {
+{% if not is_global %}
+  {{_self.decl_struct(o.req)}}
+  {{_self.decl_struct(o.rsp)}}
+{% endif %}
+
   public IRpcError error = null;
   public {{o.req.name}} req = new {{o.req.name}}();
   public {{o.rsp.name}} rsp = new {{o.rsp.name}}();
@@ -421,6 +450,7 @@ public class {{o.name}} : IRpc
 {% endmacro %}
 
 {%- macro diff_methods_for(u) -%}
+  {% if has_token(u, 'table') %}
   static public bool DiffOne(ref {{u.name}} curr, {{u.name}} old)
   {
     return !{{_self.compare_func_name(u)}}(ref curr, old);
@@ -451,7 +481,7 @@ public class {{o.name}} : IRpc
       }
       else if(old > item)
       {
-        item.SetDirtyMask();
+        item.SetDirtyMaskDeep();
         if(changed != null)
           changed.Add(item);
         i++;
@@ -475,7 +505,7 @@ public class {{o.name}} : IRpc
     for(; i < items.Count; i++)
     {
       var item = items[i];
-      item.SetDirtyMask();
+      item.SetDirtyMaskDeep();
       if(changed != null)
         changed.Add(item);
       has_diff = true;
@@ -496,6 +526,7 @@ public class {{o.name}} : IRpc
 
     return has_diff;
   }
+  {% endif %}
 
   public static bool {{_self.compare_func_name(u)}}(ref {{u.name}} a, {{u.name}} b)
   {
@@ -523,27 +554,121 @@ public class {{o.name}} : IRpc
 {{has_token(o, 'bitfields')?'CompareAndMarkFields':'IsEqual'}}
 {%- endmacro %}
 
-{%- macro field_compare(o, f, field_idx) -%}
+{% macro field_compare_array_bitfields_nested_diffable(o, f, field_idx) %}
+    {
+      int {{f.name}}_count_a = a.{{f.name}} == null ? 0 : a.{{f.name}}.Count;
+      int {{f.name}}_count_b = b.{{f.name}} == null ? 0 : b.{{f.name}}.Count;
 
-  {% if f.type is instanceof('\\mtgArrType') ~%}
+      //For NESTED arrays of DIFFABLE structs:
+      //for DB compatibility we must add the WHOLE ARRAY to the diff if ANY ELEMENT of the array has changed.
+      //That means marking each element's fields_mask as dirty
+
+      //Changed size means the array has changed, no furhter checks required.
+      bool {{f.name}}_equal = {{f.name}}_count_a == {{f.name}}_count_b;
+
+      //For same size arrays - check contents for changes.
+      if({{f.name}}_equal)
+      {
+        for(int i=0;i<{{f.name}}_count_a && i<{{f.name}}_count_b;++i)
+        {
+          var tmp_{{f.name}} = a.{{f.name}}[i];
+          if(!CompareAndMarkFields(ref tmp_{{f.name}}, b.{{f.name}}[i]))
+          {
+            {{f.name}}_equal = false;
+            break;
+          }
+        }
+      }
+
+      //If change detected - mark top-level field as changed and add the whole array to the diff
+      if(!{{f.name}}_equal)
+      {
+        MetaIO.SetFieldDirty(ref a.fields_mask, {{field_idx}});
+        for(int i=0; i<{{f.name}}_count_a;++i)
+        {
+          var tmp_{{f.name}} = a.{{f.name}}[i];
+          tmp_{{f.name}}.SetDirtyMaskDeep();
+          a.{{f.name}}[i] = tmp_{{f.name}};
+          is_equal = false;
+        }
+      }
+    }
+{%- endmacro %}
+
+{% macro field_compare_array_bitfields_nested_plain(o, f, field_idx) %}
+    if(a.{{f.name}} == null ||
+       b.{{f.name}} == null ||
+       a.{{f.name}}.Count != b.{{f.name}}.Count)
+    {
+      MetaIO.SetFieldDirty(ref a.fields_mask, {{field_idx}});
+      is_equal = false;
+    }
+    else
+    {
+      for(int i=0;i<a.{{f.name}}.Count;++i)
+      {
+        {%- if f.type.value is instanceof('\\mtgMetaStruct') ~%}
+          var tmp_{{f.name}} = a.{{f.name}}[i];
+          if(!{{_self.compare_func_name(f.type.value)}}(ref tmp_{{f.name}}, b.{{f.name}}[i]))
+          {
+            MetaIO.SetFieldDirty(ref a.fields_mask, {{field_idx}});
+            is_equal = false;
+            break;
+          }
+        {%- else -%}
+          if(a.{{f.name}}[i] != b.{{f.name}}[i])
+          {
+            MetaIO.SetFieldDirty(ref a.fields_mask, {{field_idx}});
+            is_equal = false;
+            break;
+          }
+        {%- endif -%}
+      }
+    }
+{%- endmacro %}
+
+{% macro field_compare_array_plain(o, f, field_idx) %}
     if(a.{{f.name}} == null || 
        b.{{f.name}} == null || 
        a.{{f.name}}.Count != b.{{f.name}}.Count) 
       return false;
-
     for(int i=0;i<a.{{f.name}}.Count;++i) 
     {
       {%- if f.type.value is instanceof('\\mtgMetaStruct') ~%}
-        if(!IsEqual(ref a.{{f.name}}[i], b.{{f.name}}[i])) 
+        var tmp_{{f.name}} = a.{{f.name}}[i];
+        if(!{{_self.compare_func_name(f.type.value)}}(ref tmp_{{f.name}}, b.{{f.name}}[i]))
           return false;
       {%- else -%}
-        if(a.{{f.name}}[i] != b.{{f.name}}[i]) 
+        if(a.{{f.name}}[i] != b.{{f.name}}[i])
           return false;
       {%- endif -%}
     }
+{%- endmacro %}
+
+{%- macro field_compare(o, f, field_idx) -%}
+
+  {% if f.type is instanceof('\\mtgArrType') ~%}
+    {% if has_token(o, 'bitfields') ~%}
+      {% if has_token(f.type.value, 'bitfields') ~%}
+        {{_self.field_compare_array_bitfields_nested_diffable(o, f, field_idx)}}
+      {% else ~%}
+        {{_self.field_compare_array_bitfields_nested_plain(o, f, field_idx)}}
+      {% endif ~%}
+    {% else ~%}
+      {{_self.field_compare_array_plain(o, f, field_idx)}}
+    {% endif ~%}
+
   {% elseif f.type is instanceof('\\mtgMetaStruct') ~%}
-    if(!IsEqual(ref a.{{f.name}}, b.{{f.name}})) 
+    var tmp_{{f.name}} = a.{{f.name}}[i];
+    if(!{{_self.compare_func_name(f.type.value)}}(ref tmp_{{f.name}}, b.{{f.name}}))
+    {% if has_token(o, 'bitfields') ~%}
+    {
+      MetaIO.SetFieldDirty(ref a.fields_mask, {{field_idx}});
+      is_equal = false;
+    }
+    {% else ~%}
       return false;
+    {% endif ~%}
   {% elseif f.type is instanceof('\\mtgBuiltinType') and f.type.isstring ~%}
       if((a.{{f.name}} == null ? "" : a.{{f.name}}) 
       != (b.{{f.name}} == null ? "" : b.{{f.name}}))
@@ -670,4 +795,3 @@ public interface {{ai.interfacename}}
 {% endfor %}
 {%- endfor ~%}
 {% endmacro %}
-