Locale Tests (#1079)

* Locale tests redo

* Add helper

* Remove line break if found.

* Fix imports

* Remove copy that snuck in

* Install language packs

* Change language pack install

* fix china language pack

* Fix tests that I accidentially broke

* Add debug print

* Don't try and parse Travis cache dirs

* Fix counting

.travis.yml

@@ -8,6 +8,7 @@ addons:
   apt:
     packages:
 before_install:
+  - sudo apt-get update && sudo apt-get --reinstall install -qq language-pack-en language-pack-ru language-pack-he language-pack-zh-hans
   - pip install tox
   # We're not actually installing Tox, but have to run it before we install wxPython via Conda. This is fugly but vOv
   - tox

_development/helpers_locale.py

@@ -0,0 +1,101 @@
+import os
+
+# https://msdn.microsoft.com/en-us/library/windows/desktop/dd317756(v=vs.85).aspx
+windows_codecs = {
+    'cp1252',  # Standard Windows
+    'cp1251',  # Russian
+    'cp037',
+    'cp424',
+    'cp437',
+    'cp500',
+    'cp720',
+    'cp737',
+    'cp775',
+    'cp850',
+    'cp852',
+    'cp855',
+    'cp856',
+    'cp857',
+    'cp858',
+    'cp860',
+    'cp861',
+    'cp862',
+    'cp863',
+    'cp864',
+    'cp865',
+    'cp866',
+    'cp869',
+    'cp874',
+    'cp875',
+    'cp932',
+    'cp949',
+    'cp950',
+    'cp1006',
+    'cp1026',
+    'cp1140',
+    'cp1250',
+    'cp1253',
+    'cp1254',
+    'cp1255',
+    'cp1256',
+    'cp1257',
+    'cp1258',
+}
+
+linux_codecs = {
+    'utf_8',  # Generic Linux/Mac
+}
+
+mac_codecs = [
+    'utf_8',  # Generic Linux/Mac
+    'mac_cyrillic',
+    'mac_greek',
+    'mac_iceland',
+    'mac_latin2',
+    'mac_roman',
+    'mac_turkish',
+]
+
+universal_codecs = [
+    'utf_16', 'utf_32', 'utf_32_be', 'utf_32_le', 'utf_16_be', 'utf_16_le', 'utf_7', 'utf_8_sig',
+]
+
+other_codecs = [
+    'scii', 'big5', 'big5hkscs', 'euc_jp', 'euc_jis_2004', 'euc_jisx0213', 'euc_kr', 'gb2312', 'gbk', 'gb18030', 'hz', 'iso2022_jp', 'iso2022_jp_1',
+    'iso2022_jp_2', 'iso2022_jp_2004', 'iso2022_jp_3', 'iso2022_jp_ext', 'iso2022_kr', 'latin_1', 'iso8859_2', 'iso8859_3', 'iso8859_4', 'iso8859_5',
+    'iso8859_6', 'iso8859_7', 'iso8859_8', 'iso8859_9', 'iso8859_10', 'iso8859_11', 'iso8859_13', 'iso8859_14', 'iso8859_15', 'iso8859_16', 'johab', 'koi8_r',
+    'koi8_u', 'ptcp154', 'shift_jis', 'shift_jis_2004', 'shift_jisx0213'
+]
+
+system_names = {
+    'Windows': windows_codecs,
+    'Linux': linux_codecs,
+    'Darwin': mac_codecs,
+}
+
+
+def GetPath(root, file=None, codec=None):
+    # Replace this with the function we actually use for this
+    path = os.path.realpath(os.path.abspath(root))
+
+    if file:
+        path = os.path.join(path, file)
+
+    if codec:
+        path = path.decode(codec)
+
+    return path
+
+def GetUnicodePath(root, file=None, codec=None):
+    # Replace this with the function we actually use for this
+    path = os.path.realpath(os.path.abspath(root))
+
+    if file:
+        path = os.path.join(path, file)
+
+    if codec:
+        path = unicode(path, codec)
+    else:
+        path = unicode(path)
+
+    return path

tests/test_locale/file_dialog.py

@@ -0,0 +1,93 @@
+# noinspection PyPackageRequirements
+import wx
+import sys
+import os
+import sys
+
+script_dir = os.path.dirname(os.path.abspath(__file__))
+# Add root to python paths, this allows us to import submodules
+sys.path.append(os.path.realpath(os.path.join(script_dir, '..', '..')))
+
+from _development.helpers_locale import GetPath, GetUnicodePath
+
+class MyForm(wx.Frame):
+    # ----------------------------------------------------------------------
+    def __init__(self):
+        wx.Frame.__init__(self, None, wx.ID_ANY, "CTRL-O to open, CTRL-S to save", size=(500, 500))
+
+        # Add a panel so it looks the correct on all platforms
+        panel = wx.Panel(self, wx.ID_ANY)
+
+        SAVE_FILE_ID = wx.NewId()
+        self.Bind(wx.EVT_MENU, self.saveFile, id=SAVE_FILE_ID)
+
+        LOAD_FILE_ID = wx.NewId()
+        self.Bind(wx.EVT_MENU, self.loadFile, id=LOAD_FILE_ID)
+
+        accel_tbl = wx.AcceleratorTable([(wx.ACCEL_CTRL, ord('O'), LOAD_FILE_ID),
+                                         (wx.ACCEL_CTRL, ord('S'), SAVE_FILE_ID)]
+                                        )
+        self.SetAcceleratorTable(accel_tbl)
+
+    # ----------------------------------------------------------------------
+    def loadFile(self, event):
+        openFileDialog = wx.FileDialog(self, "Open", "", "",
+                                       "Python files (*.py)|*.py",
+                                       wx.FD_OPEN | wx.FD_FILE_MUST_EXIST)
+        openFileDialog.ShowModal()
+        path = openFileDialog.GetPath()
+        try:
+            os_walk_without_codec = GetPath(path)
+        except (UnicodeEncodeError, UnicodeTranslateError, UnicodeError, UnicodeDecodeError, UnicodeWarning, TypeError) as e:
+            os_walk_without_codec = e
+
+        try:
+            os_walk_with_system_codec = GetPath(path, None, sys.getdefaultencoding())
+        except (UnicodeEncodeError, UnicodeTranslateError, UnicodeError, UnicodeDecodeError, UnicodeWarning, TypeError) as e:
+            os_walk_with_system_codec = e
+
+        try:
+            os_walk_unicode_without_codec = GetUnicodePath(path)
+        except (UnicodeEncodeError, UnicodeTranslateError, UnicodeError, UnicodeDecodeError, UnicodeWarning, TypeError) as e:
+            os_walk_unicode_without_codec = e
+
+        try:
+            os_walk_unicode_with_system_codec = GetUnicodePath(path, None, sys.getdefaultencoding())
+        except (UnicodeEncodeError, UnicodeTranslateError, UnicodeError, UnicodeDecodeError, UnicodeWarning, TypeError) as e:
+            os_walk_unicode_with_system_codec = e
+
+        print("Simple print:")
+        print(path)
+
+        print("Type:")
+        print(type(path))
+
+        print("OS Walk: No Codec:")
+        print(os_walk_without_codec)
+
+        print("OS Walk: Default System Codec:")
+        print(os_walk_with_system_codec)
+
+        print("OS Unicode Walk: No Codec:")
+        print(os_walk_unicode_without_codec)
+
+        print("OS Unicode Walk: Default System Codec:")
+        print(os_walk_unicode_with_system_codec)
+        openFileDialog.Destroy()
+
+    # ----------------------------------------------------------------------
+    def saveFile(self, event):
+        saveFileDialog = wx.FileDialog(self, "Save As", "", "",
+                                       "Python files (*.py)|*.py",
+                                       wx.FD_SAVE | wx.FD_OVERWRITE_PROMPT)
+        saveFileDialog.ShowModal()
+        saveFileDialog.GetPath()
+        saveFileDialog.Destroy()
+
+
+# Run the program
+if __name__ == "__main__":
+    app = wx.App(False)
+    frame = MyForm()
+    frame.Show()
+    app.MainLoop()

tests/test_locale/readme.md

@@ -0,0 +1 @@
+Use this to dynamically test languages.

tests/test_locale/test_Pyfa/test_codec_english.py

@@ -0,0 +1,36 @@
+# English
+
+import os
+import platform
+import sys
+
+script_dir = os.path.dirname(os.path.abspath(__file__))
+# Add root to python paths, this allows us to import submodules
+sys.path.append(os.path.realpath(os.path.join(script_dir, '..', '..', '..')))
+
+from _development.helpers_locale import GetPath
+
+
+def test_codec_english():
+    use_codec = {
+        "Windows": "cp1252",
+        "Linux"  : "utf8",
+        "Darwin" : "utf8",
+    }
+
+    os_name = platform.system()
+    current_directory = os.path.dirname(os.path.abspath(__file__))
+
+    try:
+        decoded_file = GetPath(current_directory, "testcodec", use_codec[os_name])
+    except:
+        assert False, "Specified codec (" + use_codec[os_name] + ") failed to decrypt file path."
+
+    try:
+        with open(decoded_file, 'r') as f:
+            read_data = f.read()
+        f.closed
+    except:
+        assert False, "Specified codec (" + use_codec[os_name] + ") failed to read file."
+
+    assert read_data == "True"

tests/test_locale/test_Pyfa/testcodec

@@ -0,0 +1 @@
+True

tests/test_locale/test_os_walk.py

@@ -0,0 +1,40 @@
+import os
+import sys
+
+script_dir = os.path.dirname(os.path.abspath(__file__))
+# Add root to python paths, this allows us to import submodules
+sys.path.append(os.path.realpath(os.path.join(script_dir, '..', '..')))
+
+from _development.helpers_locale import GetPath
+
+def test_os_walk():
+    current_directory = os.path.dirname(os.path.abspath(unicode(__file__)))
+    subfolders = os.listdir(current_directory)
+    subfolders = [e for e in subfolders if not (e.endswith(".py") or e.endswith(".pyc") or e.endswith(".md"))]
+
+    subfolder_count = 0
+    for subfolder in subfolders:
+        subdir = GetPath(current_directory, subfolder)
+        testfile = GetPath(subdir, "testcodec")
+
+        if "__pycache__" in testfile:
+            # Grabbed a Travis temp folder, skip any assertions, but count it.
+            subfolder_count += 1
+            continue
+
+        # noinspection PyBroadException
+        try:
+            with open(testfile, 'r') as f:
+                read_data = f.read()
+            # noinspection PyStatementEffect
+            f.closed
+        except:
+            print("Test File:")
+            print(testfile)
+            assert False, "Failed to read file."
+
+        read_data = read_data.replace("\n", "")
+        assert read_data == "True"
+        subfolder_count += 1
+
+    assert len(subfolders) == subfolder_count

tests/test_locale/test_знаф/test_codec_russian.py

@@ -0,0 +1,36 @@
+# Russian
+
+import os
+import platform
+import sys
+
+script_dir = os.path.dirname(os.path.abspath(__file__))
+# Add root to python paths, this allows us to import submodules
+sys.path.append(os.path.realpath(os.path.join(script_dir, '..', '..', '..')))
+
+from _development.helpers_locale import GetPath
+
+
+def test_codec_russian():
+    use_codec = {
+        "Windows": "cp1251",
+        "Linux"  : "utf8",
+        "Darwin" : "mac_cyrillic",
+    }
+
+    os_name = platform.system()
+    current_directory = os.path.dirname(os.path.abspath(__file__))
+
+    try:
+        decoded_file = GetPath(current_directory, "testcodec", use_codec[os_name])
+    except:
+        assert False, "Specified codec (" + use_codec[os_name] + ") failed to decrypt file path."
+
+    try:
+        with open(decoded_file, 'r') as f:
+            read_data = f.read()
+        f.closed
+    except:
+        assert False, "Specified codec (" + use_codec[os_name] + ") failed to read file."
+
+    assert read_data == "True"

tests/test_locale/test_знаф/testcodec

@@ -0,0 +1 @@
+True

tests/test_locale/test_פטכש/test_codec_hebrew.py

@@ -0,0 +1,36 @@
+# Hebrew
+
+import os
+import platform
+import sys
+
+script_dir = os.path.dirname(os.path.abspath(__file__))
+# Add root to python paths, this allows us to import submodules
+sys.path.append(os.path.realpath(os.path.join(script_dir, '..', '..', '..')))
+
+from _development.helpers_locale import GetPath
+
+
+def test_codec_hebrew():
+    use_codec = {
+        "Windows": "cp1252",
+        "Linux"  : "utf8",
+        "Darwin" : "utf8",
+    }
+
+    os_name = platform.system()
+    current_directory = os.path.dirname(os.path.abspath(__file__))
+
+    try:
+        decoded_file = GetPath(current_directory, "testcodec", use_codec[os_name])
+    except:
+        assert False, "Specified codec (" + use_codec[os_name] + ") failed to decrypt file path."
+
+    try:
+        with open(decoded_file, 'r') as f:
+            read_data = f.read()
+        f.closed
+    except:
+        assert False, "Specified codec (" + use_codec[os_name] + ") failed to read file."
+
+    assert read_data == "True"

tests/test_locale/test_פטכש/testcodec

@@ -0,0 +1 @@
+True

tests/test_locale/test_测试/test_codec_chinese_simplified.py

@@ -0,0 +1,36 @@
+# Chinese (Simplified)
+
+import os
+import platform
+import sys
+
+script_dir = os.path.dirname(os.path.abspath(__file__))
+# Add root to python paths, this allows us to import submodules
+sys.path.append(os.path.realpath(os.path.join(script_dir, '..', '..', '..')))
+
+from _development.helpers_locale import GetPath
+
+
+def test_codec_chinese_simplified():
+    use_codec = {
+        "Windows": "cp1252",
+        "Linux"  : "utf8",
+        "Darwin" : "utf8",
+    }
+
+    os_name = platform.system()
+    current_directory = os.path.dirname(os.path.abspath(__file__))
+
+    try:
+        decoded_file = GetPath(current_directory, "testcodec", use_codec[os_name])
+    except:
+        assert False, "Specified codec (" + use_codec[os_name] + ") failed to decrypt file path."
+
+    try:
+        with open(decoded_file, 'r') as f:
+            read_data = f.read()
+        f.closed
+    except:
+        assert False, "Specified codec (" + use_codec[os_name] + ") failed to read file."
+
+    assert read_data == "True"

tests/test_locale/test_测试/testcodec

@@ -0,0 +1 @@
+True