jellyfin/Emby.Server.Implementations/TextEncoding/NLangDetect/Utils/NGram.cs

// TODO IMM HI: check which classes can be made internal?

using System.Collections.Generic;
using System.Text;
using NLangDetect.Core.Extensions;

namespace NLangDetect.Core.Utils
{
    public class NGram
    {
        public const int GramsCount = 3;

        private static readonly string Latin1Excluded = Messages.getString("NGram.LATIN1_EXCLUDE");

        private static readonly string[] CjkClass =
          {
        #region CJK classes

        Messages.getString("NGram.KANJI_1_0"),
        Messages.getString("NGram.KANJI_1_2"),
        Messages.getString("NGram.KANJI_1_4"),
        Messages.getString("NGram.KANJI_1_8"),
        Messages.getString("NGram.KANJI_1_11"),
        Messages.getString("NGram.KANJI_1_12"),
        Messages.getString("NGram.KANJI_1_13"),
        Messages.getString("NGram.KANJI_1_14"),
        Messages.getString("NGram.KANJI_1_16"),
        Messages.getString("NGram.KANJI_1_18"),
        Messages.getString("NGram.KANJI_1_22"),
        Messages.getString("NGram.KANJI_1_27"),
        Messages.getString("NGram.KANJI_1_29"),
        Messages.getString("NGram.KANJI_1_31"),
        Messages.getString("NGram.KANJI_1_35"),
        Messages.getString("NGram.KANJI_2_0"),
        Messages.getString("NGram.KANJI_2_1"),
        Messages.getString("NGram.KANJI_2_4"),
        Messages.getString("NGram.KANJI_2_9"),
        Messages.getString("NGram.KANJI_2_10"),
        Messages.getString("NGram.KANJI_2_11"),
        Messages.getString("NGram.KANJI_2_12"),
        Messages.getString("NGram.KANJI_2_13"),
        Messages.getString("NGram.KANJI_2_15"),
        Messages.getString("NGram.KANJI_2_16"),
        Messages.getString("NGram.KANJI_2_18"),
        Messages.getString("NGram.KANJI_2_21"),
        Messages.getString("NGram.KANJI_2_22"),
        Messages.getString("NGram.KANJI_2_23"),
        Messages.getString("NGram.KANJI_2_28"),
        Messages.getString("NGram.KANJI_2_29"),
        Messages.getString("NGram.KANJI_2_30"),
        Messages.getString("NGram.KANJI_2_31"),
        Messages.getString("NGram.KANJI_2_32"),
        Messages.getString("NGram.KANJI_2_35"),
        Messages.getString("NGram.KANJI_2_36"),
        Messages.getString("NGram.KANJI_2_37"),
        Messages.getString("NGram.KANJI_2_38"),
        Messages.getString("NGram.KANJI_3_1"),
        Messages.getString("NGram.KANJI_3_2"),
        Messages.getString("NGram.KANJI_3_3"),
        Messages.getString("NGram.KANJI_3_4"),
        Messages.getString("NGram.KANJI_3_5"),
        Messages.getString("NGram.KANJI_3_8"),
        Messages.getString("NGram.KANJI_3_9"),
        Messages.getString("NGram.KANJI_3_11"),
        Messages.getString("NGram.KANJI_3_12"),
        Messages.getString("NGram.KANJI_3_13"),
        Messages.getString("NGram.KANJI_3_15"),
        Messages.getString("NGram.KANJI_3_16"),
        Messages.getString("NGram.KANJI_3_18"),
        Messages.getString("NGram.KANJI_3_19"),
        Messages.getString("NGram.KANJI_3_22"),
        Messages.getString("NGram.KANJI_3_23"),
        Messages.getString("NGram.KANJI_3_27"),
        Messages.getString("NGram.KANJI_3_29"),
        Messages.getString("NGram.KANJI_3_30"),
        Messages.getString("NGram.KANJI_3_31"),
        Messages.getString("NGram.KANJI_3_32"),
        Messages.getString("NGram.KANJI_3_35"),
        Messages.getString("NGram.KANJI_3_36"),
        Messages.getString("NGram.KANJI_3_37"),
        Messages.getString("NGram.KANJI_3_38"),
        Messages.getString("NGram.KANJI_4_0"),
        Messages.getString("NGram.KANJI_4_9"),
        Messages.getString("NGram.KANJI_4_10"),
        Messages.getString("NGram.KANJI_4_16"),
        Messages.getString("NGram.KANJI_4_17"),
        Messages.getString("NGram.KANJI_4_18"),
        Messages.getString("NGram.KANJI_4_22"),
        Messages.getString("NGram.KANJI_4_24"),
        Messages.getString("NGram.KANJI_4_28"),
        Messages.getString("NGram.KANJI_4_34"),
        Messages.getString("NGram.KANJI_4_39"),
        Messages.getString("NGram.KANJI_5_10"),
        Messages.getString("NGram.KANJI_5_11"),
        Messages.getString("NGram.KANJI_5_12"),
        Messages.getString("NGram.KANJI_5_13"),
        Messages.getString("NGram.KANJI_5_14"),
        Messages.getString("NGram.KANJI_5_18"),
        Messages.getString("NGram.KANJI_5_26"),
        Messages.getString("NGram.KANJI_5_29"),
        Messages.getString("NGram.KANJI_5_34"),
        Messages.getString("NGram.KANJI_5_39"),
        Messages.getString("NGram.KANJI_6_0"),
        Messages.getString("NGram.KANJI_6_3"),
        Messages.getString("NGram.KANJI_6_9"),
        Messages.getString("NGram.KANJI_6_10"),
        Messages.getString("NGram.KANJI_6_11"),
        Messages.getString("NGram.KANJI_6_12"),
        Messages.getString("NGram.KANJI_6_16"),
        Messages.getString("NGram.KANJI_6_18"),
        Messages.getString("NGram.KANJI_6_20"),
        Messages.getString("NGram.KANJI_6_21"),
        Messages.getString("NGram.KANJI_6_22"),
        Messages.getString("NGram.KANJI_6_23"),
        Messages.getString("NGram.KANJI_6_25"),
        Messages.getString("NGram.KANJI_6_28"),
        Messages.getString("NGram.KANJI_6_29"),
        Messages.getString("NGram.KANJI_6_30"),
        Messages.getString("NGram.KANJI_6_32"),
        Messages.getString("NGram.KANJI_6_34"),
        Messages.getString("NGram.KANJI_6_35"),
        Messages.getString("NGram.KANJI_6_37"),
        Messages.getString("NGram.KANJI_6_39"),
        Messages.getString("NGram.KANJI_7_0"),
        Messages.getString("NGram.KANJI_7_3"),
        Messages.getString("NGram.KANJI_7_6"),
        Messages.getString("NGram.KANJI_7_7"),
        Messages.getString("NGram.KANJI_7_9"),
        Messages.getString("NGram.KANJI_7_11"),
        Messages.getString("NGram.KANJI_7_12"),
        Messages.getString("NGram.KANJI_7_13"),
        Messages.getString("NGram.KANJI_7_16"),
        Messages.getString("NGram.KANJI_7_18"),
        Messages.getString("NGram.KANJI_7_19"),
        Messages.getString("NGram.KANJI_7_20"),
        Messages.getString("NGram.KANJI_7_21"),
        Messages.getString("NGram.KANJI_7_23"),
        Messages.getString("NGram.KANJI_7_25"),
        Messages.getString("NGram.KANJI_7_28"),
        Messages.getString("NGram.KANJI_7_29"),
        Messages.getString("NGram.KANJI_7_32"),
        Messages.getString("NGram.KANJI_7_33"),
        Messages.getString("NGram.KANJI_7_35"),
        Messages.getString("NGram.KANJI_7_37"),

        #endregion
      };

        private static readonly Dictionary<char, char> _cjkMap;

        private StringBuilder _grams;
        private bool _capitalword;

        #region Constructor(s)

        static NGram()
        {
            _cjkMap = new Dictionary<char, char>();

            foreach (string cjk_list in CjkClass)
            {
                char representative = cjk_list[0];

                for (int i = 0; i < cjk_list.Length; i++)
                {
                    _cjkMap.Add(cjk_list[i], representative);
                }
            }
        }

        public NGram()
        {
            _grams = new StringBuilder(" ");
            _capitalword = false;
        }

        #endregion

        #region Public methods

        public static char Normalize(char ch)
        {
            UnicodeBlock? unicodeBlock = ch.GetUnicodeBlock();

            if (!unicodeBlock.HasValue)
            {
                return ch;
            }

            switch (unicodeBlock.Value)
            {
                case UnicodeBlock.BasicLatin:
                    {
                        if (ch < 'A' || (ch < 'a' && ch > 'Z') || ch > 'z')
                        {
                            return ' ';
                        }

                        break;
                    }

                case UnicodeBlock.Latin1Supplement:
                    {
                        if (Latin1Excluded.IndexOf(ch) >= 0)
                        {
                            return ' ';
                        }

                        break;
                    }

                case UnicodeBlock.GeneralPunctuation:
                    {
                        return ' ';
                    }

                case UnicodeBlock.Arabic:
                    {
                        if (ch == '\u06cc')
                        {
                            return '\u064a';
                        }

                        break;
                    }

                case UnicodeBlock.LatinExtendedAdditional:
                    {
                        if (ch >= '\u1ea0')
                        {
                            return '\u1ec3';
                        }

                        break;
                    }

                case UnicodeBlock.Hiragana:
                    {
                        return '\u3042';
                    }

                case UnicodeBlock.Katakana:
                    {
                        return '\u30a2';
                    }

                case UnicodeBlock.Bopomofo:
                case UnicodeBlock.BopomofoExtended:
                    {
                        return '\u3105';
                    }

                case UnicodeBlock.CjkUnifiedIdeographs:
                    {
                        if (_cjkMap.ContainsKey(ch))
                        {
                            return _cjkMap[ch];
                        }

                        break;
                    }

                case UnicodeBlock.HangulSyllables:
                    {
                        return '\uac00';
                    }
            }

            return ch;
        }

        public void AddChar(char ch)
        {
            ch = Normalize(ch);
            char lastchar = _grams[_grams.Length - 1];
            if (lastchar == ' ')
            {
                _grams = new StringBuilder(" ");
                _capitalword = false;
                if (ch == ' ') return;
            }
            else if (_grams.Length >= GramsCount)
            {
                _grams.Remove(0, 1);
            }
            _grams.Append(ch);

            if (char.IsUpper(ch))
            {
                if (char.IsUpper(lastchar)) _capitalword = true;
            }
            else
            {
                _capitalword = false;
            }
        }

        public string Get(int n)
        {
            if (_capitalword)
            {
                return null;
            }

            int len = _grams.Length;

            if (n < 1 || n > 3 || len < n)
            {
                return null;
            }

            if (n == 1)
            {
                char ch = _grams[len - 1];

                if (ch == ' ')
                {
                    return null;
                }

                return ch.ToString();
            }

            // TODO IMM HI: is ToString() here effective?
            return _grams.ToString().SubSequence(len - n, len);
        }

        #endregion
    }
}