SamsonGame/Assets/Scripts/Core/Configs.cs

using System;
using System.IO;
using System.Collections.Generic;
using metagen;
using game;

public delegate void ConfigCb(ConfEntry ce);
public delegate void ConfigCbGeneric<T>(T conf) where T : ConfBase;
public delegate bool ConfigCbGenericInterruptible<T>(T conf) where T : ConfBase;

public class ConfEntry
{
  public const int FMT_BIN        = 0;
  public const int FMT_LZ4        = 1;
  public const int FMT_FILE_REF    = 2;
  public const int FMT_LZ4_REF    = 3;

  public const byte ST_UNRESOLVED      = 0;
  public const byte ST_RESOLVED        = 1;
  public const byte ST_MARK_RELEASED   = 2;

  public uint format;
  public uint id;
  public uint class_id;
  public long stream_pos;
  public int stream_size;
  public System.Type class_type {
    get {
      return AutogenBundle.id2type(class_id);
    }
  }
  //NOTE: mostly for internal usage, should not be used directly in most cases
  public ConfBase _conf;

  public byte status;
}

public class Configs
{
  public const byte BIN_REGULAR = 1;

  Stream source;

  //for less GC pressure
  Lz4DecoderStream lz_decoder = new Lz4DecoderStream();
  MemoryStream lz_stream = new MemoryStream();
  MemoryStream lz_dst_stream = new MemoryStream();
  MemoryStream entry_stream = new MemoryStream();
  MsgPackDataReader entry_msgp_reader;

  Dictionary<uint, int> ids = new Dictionary<uint, int>();
  Dictionary<uint, int> strids = new Dictionary<uint, int>();
  List<ConfEntry> data = new List<ConfEntry>();

  int resolved = 0;

  public int Count { get { return data.Count; } }
  public int Resolved { get { return resolved; } }

  public bool strict_loading_verification = true;

  public int stats_lz4 = 0;
  public int stats_bin = 0;
  public int stats_max_buf = 0;

  public const uint MIN_GEN_ID = 1 << 29;
  static uint last_gen_id = MIN_GEN_ID;
  public static uint GenId()
  {
    return last_gen_id++;
  }

  public ConfEntry this[int i]
  {
    get {
      return data[i];
    }
  }

  static byte[] hash_bytes = new byte[256];

  static public uint ID(string id)
  {
    //NOTE: avoiding memory allocations: using static memory buffer,
    //      avoding string contacation
    System.Text.Encoding.ASCII.GetBytes(id, 1, id.Length-1, hash_bytes, 0);
    System.Text.Encoding.ASCII.GetBytes(".conf.js", 0, 8, hash_bytes, id.Length-1);
    uint hash = Crc32.Compute(hash_bytes, id.Length+8-1) & 0xFFFFFFF;
    return hash;
  }

  public uint ValidID(string strid)
  {
    var ce = FindEntry(strid);
    if(ce != null)
      return ce.id;

    Error.Verify(false, "Invalid config: {0}", strid);
    return 0;
  }

  public string StrID(uint id)
  {
    return Get<ConfBase>(id).strid;
  }

  bool Check(bool condition, string msg = "")
  {
    if(strict_loading_verification)
      Error.Verify(condition, msg);
    if(!condition)
      Log.Warn("Loading failed! "+msg);
    return condition;
  }

  public bool Load(Stream source_)
  {
    if(source != null)
      source.Close();
    source = source_;
    source.Position = 0;

    entry_msgp_reader = new MsgPackDataReader(entry_stream);

    var bin_reader = new BinaryReader(source);

    uint binary_format = 0;
    uint version = 0;
    uint header_size = 0;

    try
    {
      binary_format = bin_reader.ReadByte();
      version = bin_reader.ReadUInt32();
      header_size = bin_reader.ReadUInt32();
      Log.Info("Configs format: " + binary_format + ", version: " + version + ", header size:" + header_size);
    }
    catch(Exception e)
    {
      Check(false, "Header read error: " + e.Message);
      return false;
    }

    uint header_offset = 1 + 4 + 4 + header_size;

    if(binary_format != BIN_REGULAR)
    {
      Check(false, "Unknown format");
      return false;
    }

    var msgp_reader = new MsgPackDataReader(source);

    if(!Check(msgp_reader.BeginContainer() == 0))
      return false;

    int total_configs = 0;

    if(!Check(msgp_reader.GetContainerSize(ref total_configs) == 0))
      return false;

    Log.Info("Loading configs: " + total_configs);
    while(total_configs-- > 0)
    {
      if(!Check(msgp_reader.BeginContainer() == 0, "bad array"))
        return false;

      uint format = 0;
      if(!Check(msgp_reader.ReadU32(ref format) == 0, "bad format"))
        return false;

      if(format == ConfEntry.FMT_LZ4 || format == ConfEntry.FMT_LZ4_REF)
        stats_lz4++;
      else if(format == ConfEntry.FMT_BIN || format == ConfEntry.FMT_FILE_REF)
        stats_bin++;
      else
      {
        Check(false, "Unknown format: " + format);
        return false;
      }

      uint id = 0;
      if(!Check(msgp_reader.ReadU32(ref id) == 0, "bad id"))
        return false;
      if(id == 0)
      {
        Check(false, "0 id");
        return false;
      }

      uint strid = 0;
      if(!Check(msgp_reader.ReadU32(ref strid) == 0, "bad strid"))
        return false;

      uint class_id = 0;
      if(!Check(msgp_reader.ReadU32(ref class_id) == 0, "bad classid"))
        return false;

      uint offset = 0;
      if(!Check(msgp_reader.ReadU32(ref offset) == 0, "bad offset"))
        return false;

      int size = 0;
      if(!Check(msgp_reader.ReadI32(ref size) == 0, "bad size"))
        return false;
      if(size == 0)
      {
        Check(false, "0 size for " + id);
        return false;
      }

      var ce = new ConfEntry();
      ce._conf = null;
      ce.format = format;
      ce.id = id;
      ce.class_id = class_id;
      ce.stream_pos = header_offset + offset;
      ce.stream_size = size;

      int res = Add(id, ce, strid);
      if(res != 0)
      {
        Check(false, "Duplicate id: " + id + "(" + strid + ") " + res);
        return false;
      }

      if(!Check(msgp_reader.EndContainer() == 0))
        return false;
    }

    Log.Info("Loading configs done");
    return true;
  }

  public T FindByStrID<T>(string strid) where T : class
  {
    var ce = FindEntry(strid);
    if(ce == null)
      return null;
    return ResolveEntry(ce) as T;
  }

  public T FindByID<T>(uint id) where T : class
  {
    var ce = FindEntry(id);
    if(ce == null)
      return null;
    return ResolveEntry(ce) as T;
  }

  public T Get<T>(string strid) where T : ConfBase
  {
    T o = Find<T>(strid);
    if(o == null)
      Error.Verify(false, "Invalid config: {0}", strid);
    return o;
  }

  public T Get<T>(uint id) where T : ConfBase
  {
    T o = Find<T>(id);
    if(o == null)
      Error.Verify(false, "Invalid config: {0}", id);
    return o;
  }

  public T Find<T>(string strid) where T : ConfBase
  {
    return FindByStrID<T>(strid);
  }

  public T Find<T>(uint id) where T : ConfBase
  {
    return FindByID<T>(id);
  }

  public void Release(ConfBase c)
  {
    if(c == null)
      return;

    var ce = FindEntry(c.id);
    if(ce == null || ce._conf == null)
      return;

    ce._conf = null;
    ce.status = ConfEntry.ST_UNRESOLVED;
    --resolved;
  }

  public void MarkReleased(ConfBase c)
  {
    if(c == null)
      return;

    var ce = FindEntry(c.id);
    if(ce == null || ce._conf == null)
      return;

    ce.status = ConfEntry.ST_MARK_RELEASED;
  }

  public void PurgeMarkedToRelease()
  {
    for(int i=0;i<data.Count;++i)
    {
      var ce = data[i];
      if(ce.status == ConfEntry.ST_MARK_RELEASED)
      {
        ce._conf = null;
        ce.status = ConfEntry.ST_UNRESOLVED;
        --resolved;
      }
    }
  }

  public bool EntryExists(uint id)
  {
    return FindEntry(id) != null;
  }

  public bool EntryExists<T>(uint id, bool exact_type_matching = true)
  {
    var ce = FindEntry(id);
    if(ce == null)
      return false;

    var type = typeof(T);
    if(ce.class_type == type)
      return true;

    return !exact_type_matching && ce.class_type.IsSubclassOf(type);
  }

  public System.Type TypeOf(uint id)
  {
    ConfEntry ce = FindEntry(id);
    return ce != null ? ce.class_type : null;
  }

  ConfBase ResolveEntry(ConfEntry ce)
  {
    if(ce._conf != null)
      return ce._conf;

    //Log.Debug("RESOLVE START: " + ce.id);

    byte[] res;
    int res_len;
    ResolveToBin(ce, out res, out res_len);

    entry_stream.SetData(res, 0, res_len);
    entry_msgp_reader.setPos(0);

    FillConfigEntry(ce, entry_msgp_reader, strict_loading_verification);

    ce.status = ConfEntry.ST_RESOLVED;
    ++resolved;

    //Log.Debug("RESOLVED: " + ce._conf.strid + " " + resolved);
    return ce._conf;
  }

  public ConfBase At(int i)
  {
    return ResolveEntry(data[i]);
  }

  void ResolveToBin(ConfEntry ce, out byte[] res, out int res_len)
  {
    res = null;
    res_len = 0;

    if(ce.format == ConfEntry.FMT_BIN)
    {
      var tmp_buf = TempBuffer.Get(ce.stream_size);
      source.Position = ce.stream_pos;
      source.Read(tmp_buf, 0, ce.stream_size);

      res = tmp_buf;
      res_len = ce.stream_size;
    }
    else if(ce.format == ConfEntry.FMT_LZ4)
    {
      var lz_buf = TempBuffer.Get(ce.stream_size);
      source.Position = ce.stream_pos;
      source.Read(lz_buf, 0, ce.stream_size);

      int lz_dest_size = (int)BitConverter.ToUInt32(lz_buf, 0);
      var dst_buf = TempBuffer.Get(lz_dest_size);

      lz_dst_stream.SetData(dst_buf, 0, dst_buf.Length);
      //taking into account first 4 bytes which store uncompressed size
      lz_stream.SetData(lz_buf, 4, ce.stream_size-4);
      lz_decoder.Reset(lz_stream);
      lz_decoder.CopyToEx(lz_dst_stream);

      res = lz_dst_stream.GetBuffer();
      res_len = (int)lz_dst_stream.Position;
    }
    else if(ce.format == ConfEntry.FMT_FILE_REF)
    {
      var tmp_buf = TempBuffer.Get(ce.stream_size);
      source.Position = ce.stream_pos;
      source.Read(tmp_buf, 0, ce.stream_size);

      string file_path = System.Text.Encoding.UTF8.GetString(tmp_buf, 0, ce.stream_size);

      //NOTE: since it's a dev.only method we don't care about extra mem.allocs
      var file_bytes = File.ReadAllBytes(file_path);
      res = file_bytes;
      res_len = file_bytes.Length;
    }
    else
      Error.Assert(false, "Unknown entry format: " + ce.format);
  }

  static void FillConfigEntry(ConfEntry ce, IDataReader reader, bool strict_verification)
  {
    var mstruct = AutogenBundle.createById(ce.class_id);
    Error.Verify(mstruct != null);
    ce._conf = ReadConf(mstruct, reader, strict_verification, ce.id);
    if(ce._conf != null)
      ce._conf.id = ce.id;
  }

  static ConfBase ReadConf(IMetaStruct mstruct, IDataReader reader, bool strict_verification, uint id)
  {
    var status = Meta.SyncSafe(MetaSyncContext.NewForRead(AutogenBundle.createById, reader), ref mstruct);
    bool success = status == 0;
    if(strict_verification)
    {
      if(!success)
        Error.Verify(false, "Bad config read " + id + " " + status);
    }
    else if(!success)
      return null;

    var conf = mstruct as ConfBase;
    if(conf == null)
      Error.Verify(false, "Bad config cast " + id);
    return conf;
  }

  public int Add(uint id, ConfEntry ce, uint strid = 0)
  {
    if(ids.ContainsKey(id))
      return 1;

    if(strid != 0 && strids.ContainsKey(strid))
      return 2;

    if(strid != 0)
      strids.Add(strid, data.Count);
    ids.Add(id, data.Count);
    data.Add(ce);
    return 0;
  }

  ConfEntry FindEntry(uint id)
  {
    int idx;
    if(!ids.TryGetValue(id, out idx))
      return null;
    return data[idx];
  }

  ConfEntry FindEntry(string strid)
  {
    int idx;
    if(!strids.TryGetValue(Hash.CRC32(strid), out idx))
      return null;
    return data[idx];
  }

  public bool Replace(uint id, ConfEntry ce)
  {
    int idx;
    if(!ids.TryGetValue(id, out idx))
      return false;
    data[idx] = ce;
    return true;
  }

  public void Clear()
  {
    if(source != null)
      source.Close();
    source = null;

    ids.Clear();
    strids.Clear();
    data.Clear();

    resolved = 0;
  }

  public void Foreach(ConfigCb cb)
  {
    for(int i=0;i<Count;++i)
    {
      cb(this[i]);
    }
  }

  public void Foreach<T>(ConfigCbGeneric<T> cb) where T : ConfBase
  {
    var type = typeof(T);

    for(int i=0;i<Count;++i)
    {
      var ce = this[i];
      if(!type.IsAssignableFrom(ce.class_type))
        continue;

      var conf = Find<T>(ce.id);
      if(conf != null)
        cb(conf);
    }
  }

  public void Foreach<T>(ConfigCbGenericInterruptible<T> cb) where T : ConfBase
  {
    var type = typeof(T);

    for(int i=0;i<Count;++i)
    {
      var ce = this[i];
      if(!type.IsAssignableFrom(ce.class_type))
        continue;

      var conf = Find<T>(ce.id);
      if(conf == null)
        continue;

      if(!cb(conf))
        return;
    }
  }

  public Dictionary<string, Type> GetList()
  {
    var list = new Dictionary<string, Type>();
    foreach (ConfEntry entry in data)
    {
        list.Add(StrID(entry.id), entry.class_type);
    }

    return list;
  }
}

public static class ConfigsExtensions
{
  static public void Load(this Configs configs, Stream stream)
  {
    configs.Clear();
    configs.Load(stream);
  }
}