Bump version

Fix wxWidget crash caused by GDI object limit

.appveyor.yml

@@ -0,0 +1,186 @@
+image:
+  - Ubuntu2204
+  - Visual Studio 2022
+  - macos-catalina
+for:
+  -
+    matrix:
+      only:
+        - image: Ubuntu2204
+    environment:
+      APPVEYOR_SSH_KEY: "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDhb96UEXy8yOy/f+riX/8kKbNx/lOfIZ4pP4Cw3Gj3DmnTwEnxtRtyc+xtaxOsKbt+7+EAXFpCzYX+jHMhtd0QtWB7dbey8DBg31g0f8C5EPquqROibVbhzr/F3f6/d52FFfq6Y/CWaAvLjezvipr+zOOsIFcVusqtXdPJQ/LtUJ0LS5d4lFiw5ELHSxHIpqwGwyb7PbR3ufEFoqbr8eYiCH+vlBob72ArPfo2f3u0sMvpGYmjVVu2jj4FEY2h89sLrGyFdNWBoyumRhkb38+WSAuyPa/Y21+g+S8sRzIlkwbxicGNMtrMIi6zHEIGAgA06Sw2psP807h730PPOVaWjUcU3ojNW8hH3nPizF74pT82+iP7/fFC4PXLP+tBa+8OoHC5yiO7QKUKprMSqVa1qOm8fHbrzglplKJXfzSfUtSE+AQ+HtHhuUWKI+0LBLDrsOJwI5hbsPOAuiZ5I3VfqfAOck6SH9TcmlapVmQEypc7d7oeeUtZSOuIWKXp068= dfx@aw"
+      APPIMAGE_TOOL: appimage-builder-x86_64.AppImage
+      DEPLOY_DIR: AppDir/opt/pyfa
+      # APPVEYOR_SSH_BLOCK: true
+    cache:
+      - /home/appveyor/.cache/pip -> requirements.txt
+    # init:
+    #   - sh: curl -sflL 'https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-ssh.sh' | bash -e -
+    install:
+      - sh: sudo DEBIAN_FRONTEND=noninteractive apt-get -y update --allow-releaseinfo-change
+      # AppImage dependencies
+      - sh: sudo DEBIAN_FRONTEND=noninteractive apt-get -y install libfuse2
+      # Preparation script dependencies
+      - sh: sudo DEBIAN_FRONTEND=noninteractive apt-get -y install python3-wxgtk4.0 python3-sqlalchemy python3-logbook
+    before_build:
+      # Prepare pyfa data
+      - sh: find locale/ -type f -name "*.po" -exec msgen "{}" -o "{}" \;
+      - sh: python3 -B scripts/compile_lang.py
+      - sh: python3 -B scripts/dump_crowdin_progress.py
+      - sh: python3 -B db_update.py
+      - sh: export PYFA_VERSION="$(python3 -B scripts/dump_version.py)"
+      - sh: mkdir build
+      # Download packaging tool
+      - sh: curl -o $APPIMAGE_TOOL -L https://github.com/AppImageCrafters/appimage-builder/releases/download/v1.1.0/appimage-builder-1.1.0-x86_64.AppImage
+      - sh: chmod +x $APPIMAGE_TOOL
+    build_script:
+      - sh: mkdir -p AppDir/opt/pyfa
+      - sh: cp -r eos graphs gui imgs locale service utils eve.db config.py pyfa.py db_update.py README.md LICENSE version.yml AppDir/opt/pyfa/
+      - sh: mkdir -p AppDir/usr/share/icons/hicolor/64x64/apps/
+      - sh: cp imgs/gui/pyfa64.png AppDir/usr/share/icons/hicolor/64x64/apps/pyfa.png
+      - sh: ./$APPIMAGE_TOOL --recipe dist_assets/linux/AppImageBuilder.yml
+    after_build:
+      - sh: ls -la
+    artifacts:
+      - path: pyfa-$PYFA_VERSION-linux.AppImage
+    deploy:
+      tag: $PYFA_VERSION
+      release: pyfa $PYFA_VERSION
+      description: 'Release description'
+      provider: GitHub
+      auth_token:
+        secure: M94o0xMtzxrvlKpqMcXU2KfbJdd3aYJ3UxWzePUz/pkT1/Ojiis052CiLsLVyzJg
+      draft: true
+      force_update: false
+      # deploy on tag push only
+      on:
+        APPVEYOR_REPO_TAG: true
+  -
+    matrix:
+      only:
+        - image: Visual Studio 2022
+    environment:
+      PYTHON: "C:\\Python311-x64"
+    # Should be enabled only for build process debugging
+    # init:
+    #   - ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))
+    cache:
+      - C:\users\appveyor\appdata\local\pip\cache\ -> requirements.txt
+    install:
+      - ps: echo("OS version:")
+      - ps: "[System.Environment]::OSVersion.Version"
+
+      - ps: echo("Filesystem - root:")
+      - ps: "ls \"C:\\\""
+
+      - ps: echo("Filesystem - projects root:")
+      - ps: "ls \"C:\\projects\\\""
+
+      - ps: echo("Filesystem - pyfa root:")
+      - ps: "ls \"C:\\projects\\$env:APPVEYOR_PROJECT_SLUG\\\""
+
+      - ps: echo("Filesystem - installed SDKs:")
+      - ps: "ls \"C:\\Program Files (x86)\\Windows Kits\\\""
+
+      # Prepend newly installed Python to the PATH of this build (this cannot be
+      # done from inside the powershell script as it would require to restart
+      # the parent CMD process).
+      - cmd: "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
+      - cmd: "appveyor DownloadFile https://github.com/mlocati/gettext-iconv-windows/releases/download/v0.20.2-v1.16/gettext0.20.2-iconv1.16-shared-64.zip"
+      - cmd: "7z x gettext0.20.2-iconv1.16-shared-64.zip -ogettext"
+      - cmd: "SET PATH=gettext;%PATH%"
+
+      - cmd: "python --version"
+      - cmd: "python -c \"import struct; print(struct.calcsize('P') * 8)\""
+
+      # Upgrade to the latest version of pip to avoid it displaying warnings
+      # about it being out of date.
+      - cmd: "python -m pip install --upgrade pip"
+
+      # Install the build dependencies of the project. If some dependencies contain
+      # compiled extensions and are not provided as pre-built wheel packages,
+      # pip will build them from source using the MSVC compiler matching the
+      # target Python version and architecture
+      - ps: echo("Install pip requirements:")
+      - cmd: "python -m pip install -r requirements.txt"
+      - cmd: "python -m pip install PyInstaller==6.0.0"
+    before_build:
+        # directory that will contain the built files
+      - ps: $env:PYFA_DIST_DIR = "c:\projects\$env:APPVEYOR_PROJECT_SLUG\dist"
+      - ps: $env:PYFA_VERSION = (python ./scripts/dump_version.py)
+      - ps: echo("pyfa version $env:PYFA_VERSION")
+    build_script:
+      - ps: echo("Build pyfa:")
+      - ps: Get-ChildItem locale/*.po -Recurse -File| Foreach {msgen $_.fullname -o $_.fullname}
+        # Build language files
+      - cmd: "python scripts/compile_lang.py"
+        # Dump language progress
+      - cmd: "python scripts/dump_crowdin_progress.py"
+        # Build gamedata DB
+      - cmd: "python db_update.py"
+      # Build command for PyInstaller
+      - cmd: "python -m PyInstaller --clean -y pyfa.spec"
+      # Copy over manifest (See pyfa-org/pyfa#1622)
+      - ps: xcopy /y dist_assets\win\pyfa.exe.manifest $env:PYFA_DIST_DIR\pyfa\
+      # InnoScript EXE building. This is in a separate script because I don't feel like copying over the logic to AppVeyor script right now...
+      - cmd: "python dist_assets/win/dist.py"
+      - ps: dir $env:PYFA_DIST_DIR/
+    after_build:
+      - ps: "ls \"./\""
+      - ps: 7z a "pyfa-$env:PYFA_VERSION-win.zip" -r "$env:PYFA_DIST_DIR\pyfa\*"
+    artifacts:
+      - path: pyfa*-win.zip
+      - path: pyfa*-win.exe
+    deploy:
+      tag: $(pyfa_version)
+      release: pyfa $(pyfa_version)
+      description: 'Release description'
+      provider: GitHub
+      auth_token:
+        secure: M94o0xMtzxrvlKpqMcXU2KfbJdd3aYJ3UxWzePUz/pkT1/Ojiis052CiLsLVyzJg
+      draft: true
+      force_update: false
+      # deploy on tag push only
+      on:
+        APPVEYOR_REPO_TAG: true
+  -
+    matrix:
+      only:
+        - image: macos-catalina
+    environment:
+      APPVEYOR_SSH_KEY: "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDJDW/+oYNGOiPvwuwAL9tc/LQgg58aosIVpMYfepQZ20V+VZnHpZh8IRDA8Jo5xht19p2PksA+hFgqA0kpKtrSkuiWdE8rATQItfk4gf7yB0yGasJGGQZYazy9k/9XtmYkq2HHOOeEqdxvrICddJQ88MLCLT9lJENSUP/YS/yGcjZFXVxE11pTeIcqlCRU+3eYa1v7BeNvXIKNhZoK5orXWrtuH3cy8jrSns/u70aYfJ6B2jA8CnWnDbuvpeQtEY61SQqlKUsSArNa8NAsXj41wr3Ar9gAG9330w7EMTqlutk8HZO35uHI0q5qinUhaQYufPPrVkb2L/N+ZCfu0fnh appveyor"
+    cache:
+      - /Users/appveyor/Library/Caches/pip/ -> requirements.txt
+    init:
+    #   - sh: curl -sflL 'https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-ssh.sh' | bash -e -
+      - sh: source ~/venv3.11/bin/activate
+    install:
+      - sh: bash scripts/osx-setup.sh
+    build_script:
+      - sh: bash scripts/osx-translations.sh
+      - sh: python3 scripts/compile_lang.py
+      - sh: python3 scripts/dump_crowdin_progress.py
+      - sh: python3 db_update.py
+    after_build:
+      - sh: export PYFA_VERSION="$(python3 scripts/dump_version.py)"
+      - sh: bash scripts/osx-package.sh
+#    on_finish:
+#      - sh: export APPVEYOR_SSH_BLOCK=true
+#      - sh: curl -sflL 'https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-ssh.sh' | bash -e -
+    artifacts:
+      - path: dist/pyfa*-mac.zip
+    before_deploy:
+      - sh: export RELEASE_PKG_FILE=$(ls *.deb)
+      - sh: echo "deploying $RELEASE_PKG_FILE to GitHub releases"
+    deploy:
+      tag: $PYFA_VERSION
+      release: pyfa $PYFA_VERSION
+      description: 'Release description'
+      provider: GitHub
+      auth_token:
+        secure: M94o0xMtzxrvlKpqMcXU2KfbJdd3aYJ3UxWzePUz/pkT1/Ojiis052CiLsLVyzJg
+      draft: true
+      force_update: false
+      # deploy on tag push only
+      on:
+        APPVEYOR_REPO_TAG: true

.codecov.yml

@@ -0,0 +1,26 @@
+codecov:
+  notify:
+    require_ci_to_pass: yes
+
+coverage:
+  precision: 2
+  round: down
+  range: "70...100"
+
+  status:
+    project: yes
+    patch: yes
+    changes: no
+
+parsers:
+  gcov:
+    branch_detection:
+      conditional: yes
+      loop: yes
+      method: no
+      macro: no
+
+comment:
+  layout: "header, diff"
+  behavior: default
+  require_changes: no

.gitattributes

@@ -0,0 +1,36 @@
+# Set the default behavior, in case people don't have core.autocrlf set.
+* text=auto
+# Explicitly declare text files you want to always be normalized and converted
+# to native line endings on checkout.
+# *.c text
+# *.h text
+# Declare files that will always have CRLF line endings on checkout.
+# Source files
+# ============
+*.pxd		text eol=crlf
+*.py 		text eol=crlf
+*.py3 		text eol=crlf
+*.pyw 		text eol=crlf
+*.pyx  		text eol=crlf
+pyfa.py     text eol=lf
+# Denote all files that are truly binary and should not be modified.
+# Binary files
+# ============
+*.db		binary
+*.p 		binary
+*.pkl 		binary
+*.pyc 		binary
+*.pyd		binary
+*.pyo 		binary
+# Note: .db, .p, and .pkl files are associated
+# with the python modules ``pickle``, ``dbm.*``,
+# ``shelve``, ``marshal``, ``anydbm``, & ``bsddb``
+# (among others).
+# Denote all files that are truly binary and should not be modified.
+# Image files
+# ============
+*.png binary
+*.jpg binary
+*.icns binary
+*.ico binary
+

.gitignore

@@ -49,7 +49,6 @@ Pyfa.egg-info/
 #  Usually these files are written by a python script from a template
 #  before PyInstaller builds the exe, so as to inject date/other infos into it.
 *.manifest
-*.spec
 
 # Installer logs
 pip-log.txt
@@ -68,7 +67,6 @@ coverage.xml
 
 # Translations
 *.mo
-*.pot
 
 # Django stuff:
 *.log
@@ -92,6 +90,7 @@ target/
 
 # pyenv
 .python-version
+PyfaEnv/
 
 # celery beat schedule file
 celerybeat-schedule
@@ -118,3 +117,13 @@ ENV/
 # Pycharm project settings
 .idea
 eos.iml
+gitversion
+.version
+/.version
+*.swp
+
+*.fsdbinary
+/locale/progress.json
+
+# vscode settings
+.vscode

.mailmap

@@ -0,0 +1,14 @@
+cncfanatics <diego.duclos@gmail.com> cncfanatics <cncfanatics@titanium.(none)>
+blitzmann <holmes.ryan.90@gmail.com>
+blitzmann <holmes.ryan.90@gmail.com> blitzmann <ryan.xgamer99@gmail.com>
+blitzmann <holmes.ryan.90@gmail.com>
+blitzmann <holmes.ryan.90@gmail.com> blitzman <ryan.xgamer99@gmail.com>
+blitzmann <holmes.ryan.90@gmail.com> Ryan Holmes <ryan.holmes.90@gmail.com>
+blitzmann <holmes.ryan.90@gmail.com>
+Corollax <corollax@gmail.com> Corollax <corollax@corollax-laptop.(none)>
+Corollax <corollax@gmail.com> Corollax <corollax@corollax-N76VM.(none)>
+Mr. Nukealizer <mr.nukealizer@gmail.com> Mr. Nukealizer <MrNukealizer@users.noreply.github.com>
+DarkPhoenix <phoenix@mail.ru>
+Sakari Orisi <sakari@evefit.org>
+Will Wykeham <will@wykeham.net> Will Wykeham <will.wykeham@paconsulting.com>
+OISumeko <camerongrout@gmail.com> OISumeko <cameron@sporadic.co.nz>

CONTRIBUTING.md

@@ -0,0 +1,107 @@
+# Contribution
+
+## Requirements
+
+- Python 3.11
+- Git CLI installed
+- Python, pip and git are all available as command-line commands (add to the path if needed)
+
+Virtual environment will be created in *PyfaEnv* folder. Project will be cloned and run from the *PyfaDEV* folder. Separate virtual environment will be created so required libraries won't clutter the main python installation.
+
+> Commands and screens were created on Windows 10. Please, update all the paths according to your OS.
+
+## Setting up the project manually
+
+Clone the repository
+```
+git clone <repo> PyfaDEV
+```
+
+Create the virtual environment
+```
+python -m venv PyfaEnv
+```
+
+Activate the virtual environment
+
+```
+For cmd.exe: PyfaEnv\scripts\activate.bat
+For bash: source <venv>/Scripts/activate
+```
+> For other OS check [Python documentation](https://docs.python.org/3/library/venv.html)
+
+Install requirements for the project from *requirements.txt*
+```
+pip install -r PyfaDEV\requirements.txt
+```
+> For some Linux distributions, you may need to install separate wxPython bindings, such as `python-matplotlib-wx`
+
+Check that the libs from *requirements.txt* are installed
+```
+pip list
+```
+
+Build translations and database:
+```
+python scripts\compile_lang.py
+python db_update.py
+```
+
+Test that the project is starting properly
+```
+python PyfaDEV\pyfa.py
+```
+
+
+## Setting up the project with PyCharm/IntelliJ
+
+Install PyCharm / Other IntelliJ product with Python plugin
+
+After launching - select *Check out from Version Control* -> *GIt*
+
+![welcome](https://user-images.githubusercontent.com/54093496/66862580-d8edab00-ef99-11e9-94e2-e93d7043e620.png)
+
+Login to GitHub, paste the repo URL and select the folder to which to clone the project into, press *Clone*.
+
+![Clone](https://user-images.githubusercontent.com/54093496/66862748-38e45180-ef9a-11e9-9f68-4903baf47385.png)
+
+After process is complete, open *File* -> *Settings* -> *Project* -> *Project Interpreter*. 
+
+![Settings](https://user-images.githubusercontent.com/54093496/66862792-544f5c80-ef9a-11e9-9e0f-f64767f3f1b0.png)
+
+Press on options and add new virtual environment.
+
+![venv](https://user-images.githubusercontent.com/54093496/66862833-67622c80-ef9a-11e9-94fa-47cca0158d29.png)
+
+Open project tree view and double-click on the *requirements.txt*. Press *Install requirements*. Install all requirements.
+
+![Reqs](https://user-images.githubusercontent.com/54093496/66862870-7a74fc80-ef9a-11e9-9b18-e64be42c49b8.png)
+
+Create new *Run Configuration*. Set correct *Script path* and *Python interpreter*.
+
+![Run configuraion](https://user-images.githubusercontent.com/54093496/66862970-b4460300-ef9a-11e9-9fb4-20e24759904b.png)
+
+Check that the project is starting properly.
+
+## Running tests
+
+Switch to the proper virtual environment
+```
+For cmd.exe: PyfaEnv\scripts\activate.bat
+For bash: source <venv>/Scripts/activate
+```
+
+Install pytest 
+```
+pip install pytest  
+```
+
+Switch to pyfa directory.
+
+Run tests (any will do)
+```
+python -m pytest
+py.test
+```
+
+More information on tests can be found on appropriate [Wiki page](https://github.com/pyfa-org/Pyfa/wiki/Developers:-Writing-Tests-for-Pyfa).

ISSUE_TEMPLATE.md

@@ -0,0 +1,34 @@
+<!--
+
+Submit a bug report bug report or feature request
+
+Here you can inform pyfa developers of potential bugs or suggest features / improvements to the project. Please check
+to make sure that the bug hasn't been reported or feature requested before submitting. If you have general questions
+about the project and want to reach out to the developers personally, please check out out our [Slack]
+(https://pyfainvite.azurewebsites.net/).
+
+-->
+
+## Bug Report
+
+
+### Expected behavior:
+
+
+### Actual behavior:
+
+
+### Detailed steps to reproduce:
+
+
+### Fits involved in EFT format (Edit > To Clipboard > EFT):
+
+
+### Release or development git branch? Please note the release version or commit hash:
+
+
+### Operating system and version (eg: Windows 10, OS X 10.9, OS X 10.11, Ubuntu 16.10):
+
+
+### Other relevant information:
+

README.md

@@ -1,66 +1,58 @@
 # pyfa
 
-[![Join us on Slack!](https://pyfainvite.azurewebsites.net/badge.svg)](https://pyfainvite.azurewebsites.net/)
+[![Build Status](https://ci.appveyor.com/api/projects/status/github/pyfa-org/pyfa?branch=master&svg=true)]([https://travis-ci.org/pyfa-org/Pyfa](https://ci.appveyor.com/project/pyfa-org/pyfa))
 
-![pyfa](https://cloud.githubusercontent.com/assets/3904767/10271512/af385ef2-6ade-11e5-8f67-52b8b1e4c797.PNG)
+![pyfa](https://user-images.githubusercontent.com/275209/66119992-864be080-e5e2-11e9-994a-3a4368c9fad7.png)
 
 ## What is it?
 
-pyfa, short for **py**thon **f**itting **a**ssistant, allows you to create, experiment with, and save ship fittings without being in game. Open source and written in Python, it is available on any platform where Python 2.x and wxWidgets are available, including Windows, Mac OS X, and Linux.
+Pyfa, short for **py**thon **f**itting **a**ssistant, allows you to create, experiment with, and save ship fittings without being in game. Open source and written in Python, it is available on any platform where Python 3 and wxWidgets are available, including Windows, macOS, and Linux.
 
 ## Latest Version and Changelogs
-The latest version along with release notes can always be found on the project's [Releases](https://github.com/DarkFenX/Pyfa/releases) page. pyfa will notify you if you are running an outdated version.
+The latest version along with release notes can always be found on the project's [releases](https://github.com/pyfa-org/Pyfa/releases) page. Pyfa will notify you if you are running an outdated version.
 
 ## Installation
-Windows and OS X users are supplied self-contained builds of pyfa on the [latest releases](https://github.com/pyfa-org/Pyfa/releases/latest) page. An `.exe` installer is also available for Windows builds. Linux users can run pyfa using their distribution's Python interpreter. There is no official self-contained package for Linux, however, there are a number of third-party packages available through distribution-specific repositories.
+Windows, macOS, and Linux users are supplied self-contained builds of pyfa on the [latest releases](https://github.com/pyfa-org/Pyfa/releases/latest) page.
 
-#### OS X
-There are two different distributives for OS X: `-mac` and `-mac-deprecated`. 
+### Third Party Packages
+Please note that these packages are maintained by third-parties and are not evaluated by the pyfa developers.
 
-* `-mac`: based on wxPython 3.0.2.0 and has updated libraries. This is the recommended build.
-* `-mac-deprecated`: utilizes older binaries running on wxPython 2.8; because of this, some features are not available (currently CREST support and Attribute Overrides). Additionally, as development happens primarily on wxPython 3.0, a few GUI bugs may pop up as `-mac-deprecated` is not actively tested. However, due to some general issues with wxPython 3.0, especially on some newer OS X versions, `-mac-deprecated` is still offered for those that need it.
-
-There is also a [Homebrew](http://brew.sh) option for installing pyfa on OS X. Please note this is maintained by a third-party and is not tested by pyfa developers. Simply fire up in terminal:
+#### macOS
+Apart from the official release, there is also a [Homebrew](https://formulae.brew.sh/cask/pyfa) option for installing pyfa on macOS. Simply fire up in terminal:
 ```
-$ brew install Caskroom/cask/pyfa
+$ brew install --cask pyfa
 ```
 
-### Linux Distro-specific Packages
-The following is a list of pyfa packages available for certain distributions. Please note that these packages are maintained by third-parties and are not evaluated by the pyfa developers.
+#### Linux Distro-specific Packages
+The following is a list of pyfa packages available for certain distributions. 
 
-* Debian/Ubuntu/derivitives: https://github.com/AdamMajer/Pyfa/releases
 * Arch: https://aur.archlinux.org/packages/pyfa/
-* openSUSE: https://build.opensuse.org/package/show/home:rmk2/pyfa
-* FreeBSD: http://www.freshports.org/games/pyfa/ (see [#484](https://github.com/pyfa-org/Pyfa/issues/484) for instructions)
+* Gentoo: https://github.com/ZeroPointEnergy/gentoo-pyfa-overlay
 
-### Dependencies
-If you wish to help with development or simply need to run pyfa through a Python interpreter, the following software is required:
-
-* Python 2.7
-* `wxPython` 2.8/3.0
-* `sqlalchemy` >= 0.6
-* `dateutil`
-* `matplotlib` (for some Linux distributions you may need to install separate wxPython bindings such as `python-matplotlib-wx`)
-* `requests`
+## Contribution
+If you wish to help with development or you need to run pyfa through a Python interpreter, check out [the instructions](https://github.com/pyfa-org/Pyfa/blob/master/CONTRIBUTING.md).
 
 ## Bug Reporting
-The preferred method of reporting bugs is through the project's [GitHub Issues interface](https://github.com/pyfa-org/Pyfa/issues). Alternatively, posting a report in the [pyfa thread](http://forums.eveonline.com/default.aspx?g=posts&t=247609) on the official EVE Online forums is acceptable. Guidelines for bug reporting can be found on [this wiki page](https://github.com/DarkFenX/Pyfa/wiki/Bug-Reporting). 
+The preferred method of reporting bugs is through the project's [GitHub Issues interface](https://github.com/pyfa-org/Pyfa/issues). Alternatively, posting a report in the [pyfa thread](https://forums.eveonline.com/t/27156) on the official EVE Online forums is acceptable. Guidelines for bug reporting can be found on [this wiki page](https://github.com/pyfa-org/Pyfa/wiki/Bug-Reporting). 
 
 ## License
-pyfa is licensed under the GNU GPL v3.0, see LICENSE
+Pyfa is licensed under the GNU GPL v3.0, see LICENSE
 
 ## Resources
-* Development repository: [https://github.com/pyfa-org/Pyfa](https://github.com/pyfa-org/Pyfa)
-* XMPP conference: [pyfa@conference.jabber.org](pyfa@conference.jabber.org)
-* [EVE forum thread](http://forums.eveonline.com/default.aspx?g=posts&t=247609)
-* [EVE University guide using pyfa](http://wiki.eveuniversity.org/Guide_to_using_PYFA)
+* [Development repository](https://github.com/pyfa-org/Pyfa)
+* [EVE forum thread](https://forums.eveonline.com/t/27156)
+* [EVE University guide using pyfa](https://wiki.eveuniversity.org/PYFA)
 * [EVE Online website](http://www.eveonline.com/)
 
 ## Contacts:
+* Kadesh / DarkPhoenix
+    * GitHub: @DarkFenX
+    * EVE: Kadesh Priestess
+    * Email: phoenix@mail.ru
 * Sable Blitzmann
     * GitHub: @blitzmann
     * [TweetFleet Slack](https://www.fuzzwork.co.uk/tweetfleet-slack-invites/): @blitzmann
-    * [Gitter chat](https://gitter.im/pyfa-org/Pyfa): @ blitzmann
+    * [Gitter chat](https://gitter.im/pyfa-org/Pyfa): @blitzmann
     * Email: sable.blitzmann@gmail.com
 
 ## CCP Copyright Notice

_development/Pyfa_CodeStyle.xml

@@ -0,0 +1,12 @@
+<code_scheme name="Pyfa">
+  <option name="LINE_SEPARATOR" value="&#xD;&#xA;" />
+  <option name="RIGHT_MARGIN" value="165" />
+  <Python>
+    <option name="NEW_LINE_AFTER_COLON" value="true" />
+    <option name="DICT_NEW_LINE_AFTER_LEFT_BRACE" value="true" />
+    <option name="DICT_NEW_LINE_BEFORE_RIGHT_BRACE" value="true" />
+    <option name="USE_CONTINUATION_INDENT_FOR_ARGUMENTS" value="true" />
+    <option name="OPTIMIZE_IMPORTS_SORT_NAMES_IN_FROM_IMPORTS" value="true" />
+    <option name="OPTIMIZE_IMPORTS_JOIN_FROM_IMPORTS_WITH_SAME_SOURCE" value="true" />
+  </Python>
+</code_scheme>

_development/Pyfa_Inspections.xml

@@ -0,0 +1,61 @@
+  <profile version="1.0">
+    <option name="myName" value="Pyfa" />
+    <inspection_tool class="IgnoreUnusedEntry" enabled="false" level="UNUSED ENTRY" enabled_by_default="false" />
+    <inspection_tool class="InconsistentLineSeparators" enabled="true" level="ERROR" enabled_by_default="true" />
+    <inspection_tool class="ProblematicWhitespace" enabled="true" level="WARNING" enabled_by_default="true" />
+    <inspection_tool class="PyBehaveInspection" enabled="true" level="WARNING" enabled_by_default="true" />
+    <inspection_tool class="PyClassicStyleClassInspection" enabled="true" level="WARNING" enabled_by_default="true" />
+    <inspection_tool class="PyCompatibilityInspection" enabled="true" level="WARNING" enabled_by_default="true">
+      <option name="ourVersions">
+        <value>
+          <list size="1">
+            <item index="0" class="java.lang.String" itemvalue="2.7" />
+          </list>
+        </value>
+      </option>
+    </inspection_tool>
+    <inspection_tool class="PyMissingTypeHintsInspection" enabled="true" level="WEAK WARNING" enabled_by_default="true" />
+    <inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="WARNING" enabled_by_default="true">
+      <option name="ignoredPackages">
+        <value>
+          <list size="1">
+            <item index="0" class="java.lang.String" itemvalue="wxPython" />
+          </list>
+        </value>
+      </option>
+    </inspection_tool>
+    <inspection_tool class="PyPep8Inspection" enabled="true" level="TYPO" enabled_by_default="true">
+      <option name="ignoredErrors">
+        <list>
+          <option value="E203" />
+          <option value="E127" />
+          <option value="E128" />
+          <option value="E126" />
+        </list>
+      </option>
+    </inspection_tool>
+    <inspection_tool class="PyPep8NamingInspection" enabled="true" level="TYPO" enabled_by_default="true">
+      <option name="ignoredErrors">
+        <list>
+          <option value="N802" />
+          <option value="N806" />
+          <option value="N803" />
+          <option value="N814" />
+        </list>
+      </option>
+    </inspection_tool>
+    <inspection_tool class="PyShadowingBuiltinsInspection" enabled="true" level="WARNING" enabled_by_default="true" />
+    <inspection_tool class="PyShadowingNamesInspection" enabled="true" level="WARNING" enabled_by_default="true" />
+    <inspection_tool class="PyUnresolvedReferencesInspection" enabled="true" level="WARNING" enabled_by_default="true">
+      <option name="ignoredIdentifiers">
+        <list>
+          <option value="_" />
+        </list>
+      </option>
+    </inspection_tool>
+    <inspection_tool class="SpellCheckingInspection" enabled="false" level="TYPO" enabled_by_default="false">
+      <option name="processCode" value="true" />
+      <option name="processLiterals" value="true" />
+      <option name="processComments" value="true" />
+    </inspection_tool>
+  </profile>

_development/helpers.py

@@ -0,0 +1,148 @@
+# noinspection PyPackageRequirements
+import pytest
+
+import os
+import sys
+import threading
+
+from sqlalchemy import MetaData, create_engine
+from sqlalchemy.orm import sessionmaker
+
+script_dir = os.path.dirname(os.path.abspath(__file__))
+# Add root folder to python paths
+sys.path.append(os.path.realpath(os.path.join(script_dir, '..', '..')))
+sys._called_from_test = True
+
+# noinspection PyUnresolvedReferences,PyUnusedLocal
+@pytest.fixture
+def DBInMemory_test():
+    def rollback():
+        with sd_lock:
+            saveddata_session.rollback()
+
+
+    print("Creating database in memory")
+    from os.path import realpath, join, dirname, abspath
+
+    debug = False
+    gamedataCache = True
+    saveddataCache = True
+    gamedata_version = ""
+    gamedata_connectionstring = 'sqlite:///' + realpath(join(dirname(abspath(str(__file__))), "..", "eve.db"))
+    saveddata_connectionstring = 'sqlite:///:memory:'
+
+    class ReadOnlyException(Exception):
+        pass
+
+    if callable(gamedata_connectionstring):
+        gamedata_engine = create_engine("sqlite://", creator=gamedata_connectionstring, echo=debug)
+    else:
+        gamedata_engine = create_engine(gamedata_connectionstring, echo=debug)
+
+    gamedata_meta = MetaData()
+    gamedata_meta.bind = gamedata_engine
+    gamedata_session = sessionmaker(bind=gamedata_engine, autoflush=False, expire_on_commit=False)()
+
+    # This should be moved elsewhere, maybe as an actual query. Current, without try-except, it breaks when making a new
+    # game db because we haven't reached gamedata_meta.create_all()
+    try:
+        gamedata_version = gamedata_session.execute(
+            "SELECT `field_value` FROM `metadata` WHERE `field_name` LIKE 'client_build'"
+        ).fetchone()[0]
+    except (KeyboardInterrupt, SystemExit):
+        raise
+    except Exception as e:
+        print("Missing gamedata version.")
+        gamedata_version = None
+
+    if saveddata_connectionstring is not None:
+        if callable(saveddata_connectionstring):
+            saveddata_engine = create_engine(creator=saveddata_connectionstring, echo=debug)
+        else:
+            saveddata_engine = create_engine(saveddata_connectionstring, echo=debug)
+
+        saveddata_meta = MetaData()
+        saveddata_meta.bind = saveddata_engine
+        saveddata_session = sessionmaker(bind=saveddata_engine, autoflush=False, expire_on_commit=False)()
+    else:
+        saveddata_meta = None
+
+    # Lock controlling any changes introduced to session
+    sd_lock = threading.Lock()
+
+    # Import all the definitions for all our database stuff
+    # noinspection PyPep8
+    #from eos.db.gamedata import alphaClones, attribute, category, effect, group, icon, item, marketGroup, metaData, metaGroup, queries, traits, unit
+    # noinspection PyPep8
+    #from eos.db.saveddata import booster, cargo, character, crest, damagePattern, databaseRepair, drone, fighter, fit, implant, implantSet, miscData, module, override, price, queries, skill, targetProfile, user
+
+    # If using in memory saveddata, you'll want to reflect it so the data structure is good.
+    if saveddata_connectionstring == "sqlite:///:memory:":
+        saveddata_meta.create_all()
+
+    # Output debug info to help us troubleshoot Travis
+    print(saveddata_engine)
+    print(gamedata_engine)
+
+    helper = {
+        #'config': eos.config,
+        'gamedata_session'    : gamedata_session,
+        'saveddata_session'    : saveddata_session,
+    }
+    return helper
+
+# noinspection PyUnresolvedReferences,PyUnusedLocal
+@pytest.fixture
+def DBInMemory():
+    print("Creating database in memory")
+
+    import eos.config
+
+    import eos
+    import eos.db
+
+    # Output debug info to help us troubleshoot Travis
+    print(eos.db.saveddata_engine)
+    print(eos.db.gamedata_engine)
+
+    helper = {
+        'config': eos.config,
+        'db'    : eos.db,
+        'gamedata_session' : eos.db.gamedata_session,
+        'saveddata_session' : eos.db.saveddata_session,
+    }
+    return helper
+
+
+@pytest.fixture
+def Gamedata():
+    print("Building Gamedata")
+    from eos.gamedata import Item
+
+    helper = {
+        'Item': Item,
+    }
+    return helper
+
+
+@pytest.fixture
+def Saveddata():
+    print("Building Saveddata")
+    from eos.saveddata.ship import Ship
+    from eos.saveddata.fit import Fit
+    from eos.saveddata.character import Character
+    from eos.saveddata.module import Module
+    from eos.const import FittingModuleState
+    from eos.saveddata.citadel import Citadel
+    from eos.saveddata.booster import Booster
+
+    helper = {
+        'Structure': Citadel,
+        'Ship'     : Ship,
+        'Fit'      : Fit,
+        'Character': Character,
+        'Module'   : Module,
+        'State'    : FittingModuleState,
+        'Booster'  : Booster,
+    }
+    return helper

_development/helpers_fits.py

@@ -0,0 +1,65 @@
+import pytest
+
+# noinspection PyPackageRequirements
+
+
+# noinspection PyShadowingNames
+@pytest.fixture
+def RifterFit(DB, Gamedata, Saveddata):
+    print("Creating Rifter")
+    item = DB['gamedata_session'].query(Gamedata['Item']).filter(Gamedata['Item'].name == "Rifter").first()
+    ship = Saveddata['Ship'](item)
+    # setup fit
+    fit = Saveddata['Fit'](ship, "My Rifter Fit")
+
+    return fit
+
+
+# noinspection PyShadowingNames
+@pytest.fixture
+def KeepstarFit(DB, Gamedata, Saveddata):
+    print("Creating Keepstar")
+    item = DB['gamedata_session'].query(Gamedata['Item']).filter(Gamedata['Item'].name == "Keepstar").first()
+    ship = Saveddata['Structure'](item)
+    # setup fit
+    fit = Saveddata['Fit'](ship, "Keepstar Fit")
+
+    return fit
+
+
+# noinspection PyShadowingNames
+@pytest.fixture
+def CurseFit(DB, Gamedata, Saveddata):
+    print("Creating Curse - With Neuts")
+    item = DB['gamedata_session'].query(Gamedata['Item']).filter(Gamedata['Item'].name == "Curse").first()
+    ship = Saveddata['Ship'](item)
+    # setup fit
+    fit = Saveddata['Fit'](ship, "Curse - With Neuts")
+
+    mod = Saveddata['Module'](DB['db'].getItem("Medium Energy Neutralizer II"))
+    mod.state = Saveddata['State'].ONLINE
+
+    # Add 5 neuts
+    for _ in range(5):
+        fit.modules.append(mod)
+
+    return fit
+
+
+# noinspection PyShadowingNames
+@pytest.fixture
+def HeronFit(DB, Gamedata, Saveddata):
+    print("Creating Heron - RemoteSebo")
+    item = DB['gamedata_session'].query(Gamedata['Item']).filter(Gamedata['Item'].name == "Heron").first()
+    ship = Saveddata['Ship'](item)
+    # setup fit
+    fit = Saveddata['Fit'](ship, "Heron - RemoteSebo")
+
+    mod = Saveddata['Module'](DB['db'].getItem("Remote Sensor Booster II"))
+    mod.state = Saveddata['State'].ONLINE
+
+    # Add 5 neuts
+    for _ in range(4):
+        fit.modules.append(mod)
+
+    return fit

_development/helpers_items.py

@@ -0,0 +1,11 @@
+import pytest
+
+# noinspection PyPackageRequirements
+
+
+# noinspection PyShadowingNames
+@pytest.fixture
+def StrongBluePillBooster (DB, Gamedata, Saveddata):
+    print("Creating Strong Blue Pill Booster")
+    item = DB['gamedata_session'].query(Gamedata['Item']).filter(Gamedata['Item'].name == "Strong Blue Pill Booster").first()
+    return Saveddata['Booster'](item)

_development/helpers_locale.py

@@ -0,0 +1,101 @@
+import os
+
+# https://msdn.microsoft.com/en-us/library/windows/desktop/dd317756(v=vs.85).aspx
+windows_codecs = {
+    'cp1252',  # Standard Windows
+    'cp1251',  # Russian
+    'cp037',
+    'cp424',
+    'cp437',
+    'cp500',
+    'cp720',
+    'cp737',
+    'cp775',
+    'cp850',
+    'cp852',
+    'cp855',
+    'cp856',
+    'cp857',
+    'cp858',
+    'cp860',
+    'cp861',
+    'cp862',
+    'cp863',
+    'cp864',
+    'cp865',
+    'cp866',
+    'cp869',
+    'cp874',
+    'cp875',
+    'cp932',
+    'cp949',
+    'cp950',
+    'cp1006',
+    'cp1026',
+    'cp1140',
+    'cp1250',
+    'cp1253',
+    'cp1254',
+    'cp1255',
+    'cp1256',
+    'cp1257',
+    'cp1258',
+}
+
+linux_codecs = {
+    'utf_8',  # Generic Linux/Mac
+}
+
+mac_codecs = [
+    'utf_8',  # Generic Linux/Mac
+    'mac_cyrillic',
+    'mac_greek',
+    'mac_iceland',
+    'mac_latin2',
+    'mac_roman',
+    'mac_turkish',
+]
+
+universal_codecs = [
+    'utf_16', 'utf_32', 'utf_32_be', 'utf_32_le', 'utf_16_be', 'utf_16_le', 'utf_7', 'utf_8_sig',
+]
+
+other_codecs = [
+    'scii', 'big5', 'big5hkscs', 'euc_jp', 'euc_jis_2004', 'euc_jisx0213', 'euc_kr', 'gb2312', 'gbk', 'gb18030', 'hz', 'iso2022_jp', 'iso2022_jp_1',
+    'iso2022_jp_2', 'iso2022_jp_2004', 'iso2022_jp_3', 'iso2022_jp_ext', 'iso2022_kr', 'latin_1', 'iso8859_2', 'iso8859_3', 'iso8859_4', 'iso8859_5',
+    'iso8859_6', 'iso8859_7', 'iso8859_8', 'iso8859_9', 'iso8859_10', 'iso8859_11', 'iso8859_13', 'iso8859_14', 'iso8859_15', 'iso8859_16', 'johab', 'koi8_r',
+    'koi8_u', 'ptcp154', 'shift_jis', 'shift_jis_2004', 'shift_jisx0213'
+]
+
+system_names = {
+    'Windows': windows_codecs,
+    'Linux': linux_codecs,
+    'Darwin': mac_codecs,
+}
+
+
+def GetPath(root, file=None, codec=None):
+    # Replace this with the function we actually use for this
+    path = os.path.realpath(os.path.abspath(root))
+
+    if file:
+        path = os.path.join(path, file)
+
+    if codec:
+        path = path.decode(codec)
+
+    return path
+
+def GetUnicodePath(root, file=None, codec=None):
+    # Replace this with the function we actually use for this
+    path = os.path.realpath(os.path.abspath(root))
+
+    if file:
+        path = os.path.join(path, file)
+
+    if codec:
+        path = str(path, codec)
+    else:
+        path = str(path)
+
+    return path

config.py

@@ -1,48 +1,91 @@
 import os
 import sys
+import yaml
+import wx
 
-# TODO: move all logging back to pyfa.py main loop
-# We moved it here just to avoid rebuilding windows skeleton for now (any change to pyfa.py needs it)
-import logging
-import logging.handlers
+from logbook import CRITICAL, DEBUG, ERROR, FingersCrossedHandler, INFO, Logger, NestedSetup, NullHandler, \
+    StreamHandler, TimedRotatingFileHandler, WARNING
+import hashlib
+from eos.const import FittingSlot
+
+from cryptography.fernet import Fernet
+from collections import namedtuple
+
+pyfalog = Logger(__name__)
 
 # Load variable overrides specific to distribution type
 try:
     import configforced
 except ImportError:
+    pyfalog.warning("Failed to import: configforced")
     configforced = None
 
+
 # Turns on debug mode
 debug = False
 # Defines if our saveddata will be in pyfa root or not
 saveInRoot = False
 
-# Version data
-version = "1.26.0"
-tag = "Stable"
-expansionName = "YC118.10"
-expansionVersion = "1.2"
 evemonMinVersion = "4081"
 
+minItemSearchLength = 3
+minItemSearchLengthCjk = 1
+
 pyfaPath = None
 savePath = None
 saveDB = None
 gameDB = None
+imgsZIP = None
+logPath = None
+loggingLevel = None
+logging_setup = None
+cipher = None
+clientHash = None
+experimentalFeatures = None
+version = None
+language = None
+
+ApiServer = namedtuple('ApiBase', ['name', 'sso', 'esi', 'client_id', 'callback', 'supports_auto_login'])
+supported_servers = {
+    "Tranquility": ApiServer("Tranquility", "login.eveonline.com", "esi.evetech.net", '095d8cd841ac40b581330919b49fe746', 'https://pyfa-org.github.io/Pyfa/callback', True),
+    # No point having SISI: https://developers.eveonline.com/blog/article/removing-datasource-singularity
+    # "Singularity": ApiServer("Singularity", "sisilogin.testeveonline.com", "esi.evetech.net", 'b9c3cc79448f449ab17f3aebd018842e', 'https://pyfa-org.github.io/Pyfa/callback'),
+    "Serenity": ApiServer("Serenity", "login.evepc.163.com", "ali-esi.evepc.163.com", 'bc90aa496a404724a93f41b4f4e97761', 'https://ali-esi.evepc.163.com/ui/oauth2-redirect.html', False)
+}
+
+SSO_LOGOFF_SERENITY='https://login.evepc.163.com/account/logoff'
+ESI_CACHE = 'esi_cache'
+
+LOGLEVEL_MAP = {
+    "critical": CRITICAL,
+    "error": ERROR,
+    "warning": WARNING,
+    "info": INFO,
+    "debug": DEBUG,
+}
+
+CATALOG = 'lang'
 
 
-class StreamToLogger(object):
-   """
-   Fake file-like stream object that redirects writes to a logger instance.
-   From: http://www.electricmonk.nl/log/2011/08/14/redirect-stdout-and-stderr-to-a-logger-in-python/
-   """
-   def __init__(self, logger, log_level=logging.INFO):
-      self.logger = logger
-      self.log_level = log_level
-      self.linebuf = ''
+slotColourMapDark = {
+    FittingSlot.LOW: wx.Colour(44, 36, 19),  # yellow = low slots 24/13
+    FittingSlot.MED: wx.Colour(28, 39, 51),  # blue   = mid slots 8.1/9.5
+    FittingSlot.HIGH: wx.Colour(53, 31, 34),  # red    = high slots 6.5/11.5
+    FittingSlot.RIG: '',
+    FittingSlot.SUBSYSTEM: ''}
+errColorDark = wx.Colour(70, 20, 20)
+slotColourMap = {
+    FittingSlot.LOW: wx.Colour(250, 235, 204),  # yellow = low slots
+    FittingSlot.MED: wx.Colour(188, 215, 241),  # blue   = mid slots
+    FittingSlot.HIGH: wx.Colour(235, 204, 209),  # red    = high slots
+    FittingSlot.RIG: '',
+    FittingSlot.SUBSYSTEM: ''}
+errColor = wx.Colour(204, 51, 51)
+
+
+def getClientSecret():
+    return clientHash
 
-   def write(self, buf):
-      for line in buf.rstrip().splitlines():
-         self.logger.log(self.log_level, line.rstrip())
 
 def isFrozen():
     if hasattr(sys, 'frozen'):
@@ -50,28 +93,45 @@ def isFrozen():
     else:
         return False
 
-def getPyfaRoot():
-    base = getattr(sys.modules['__main__'], "__file__", sys.executable) if isFrozen() else sys.argv[0]
-    root = os.path.dirname(os.path.realpath(os.path.abspath(base)))
-    root = unicode(root, sys.getfilesystemencoding())
-    return root
 
 def __createDirs(path):
     if not os.path.exists(path):
         os.makedirs(path)
 
-def defPaths(customSavePath):
+
+def getPyfaRoot():
+    if hasattr(sys, '_MEIPASS'):
+        return sys._MEIPASS
+    base = getattr(sys.modules['__main__'], "__file__", sys.executable) if isFrozen() else __file__
+    root = os.path.dirname(os.path.realpath(os.path.abspath(base)))
+    root = root
+    return root
+
+
+def getVersion():
+    return version
+
+
+def getDefaultSave():
+    return os.path.expanduser(os.path.join("~", ".pyfa"))
+
+
+def defPaths(customSavePath=None):
     global debug
     global pyfaPath
     global savePath
     global saveDB
     global gameDB
+    global imgsZIP
     global saveInRoot
+    global logPath
+    global cipher
+    global clientHash
+    global version
+    global experimentalFeatures
+    global language
 
-    if debug:
-        logLevel = logging.DEBUG
-    else:
-        logLevel = logging.WARN
+    pyfalog.debug("Configuring Pyfa")
 
     # The main pyfa directory which contains run.py
     # Python 2.X uses ANSI by default, so we need to convert the character encoding
@@ -79,6 +139,12 @@ def defPaths(customSavePath):
     if pyfaPath is None:
         pyfaPath = getPyfaRoot()
 
+    # Version data
+
+    with open(os.path.join(pyfaPath, "version.yml"), 'r') as file:
+        data = yaml.load(file, Loader=yaml.SafeLoader)
+        version = data['version']
+
     # Where we store the saved fits etc, default is the current users home directory
     if saveInRoot is True:
         savePath = getattr(configforced, "savePath", None)
@@ -87,36 +153,26 @@ def defPaths(customSavePath):
     else:
         savePath = getattr(configforced, "savePath", None)
         if savePath is None:
-            if customSavePath is None: # customSavePath is not overriden
-                savePath = unicode(os.path.expanduser(os.path.join("~", ".pyfa")),
-                               sys.getfilesystemencoding())
+            if customSavePath is None:  # customSavePath is not overriden
+                savePath = getDefaultSave()
             else:
                 savePath = customSavePath
 
     __createDirs(savePath)
 
-    if isFrozen():
-        os.environ["REQUESTS_CA_BUNDLE"] = os.path.join(pyfaPath, "cacert.pem")
-        os.environ["SSL_CERT_FILE"] = os.path.join(pyfaPath, "cacert.pem")
+    secret_file = os.path.join(savePath, ".secret")
+    if not os.path.exists(secret_file):
+        with open(secret_file, "wb") as _file:
+            _file.write(Fernet.generate_key())
 
-    format = '%(asctime)s %(name)-24s %(levelname)-8s %(message)s'
-    logging.basicConfig(format=format, level=logLevel)
-    handler = logging.handlers.RotatingFileHandler(os.path.join(savePath, "log.txt"), maxBytes=1000000, backupCount=3)
-    formatter = logging.Formatter(format)
-    handler.setFormatter(formatter)
-    logging.getLogger('').addHandler(handler)
+    with open(secret_file, 'rb') as fp:
+        key = fp.read()
+        clientHash = hashlib.sha3_256(key).hexdigest()
+        cipher = Fernet(key)
 
-    logging.info("Starting pyfa")
-
-    if hasattr(sys, 'frozen'):
-        stdout_logger = logging.getLogger('STDOUT')
-        sl = StreamToLogger(stdout_logger, logging.INFO)
-        sys.stdout = sl
-
-        # This interferes with cx_Freeze's own handling of exceptions. Find a way to fix this.
-        #stderr_logger = logging.getLogger('STDERR')
-        #sl = StreamToLogger(stderr_logger, logging.ERROR)
-        #sys.stderr = sl
+    # if isFrozen():
+    #    os.environ["REQUESTS_CA_BUNDLE"] = os.path.join(pyfaPath, "cacert.pem")
+    #    os.environ["SSL_CERT_FILE"] = os.path.join(pyfaPath, "cacert.pem")
 
     # The database where we store all the fits etc
     saveDB = os.path.join(savePath, "saveddata.db")
@@ -124,13 +180,119 @@ def defPaths(customSavePath):
     # The database where the static EVE data from the datadump is kept.
     # This is not the standard sqlite datadump but a modified version created by eos
     # maintenance script
-    gameDB = os.path.join(pyfaPath, "eve.db")
+    gameDB = getattr(configforced, "gameDB", gameDB)
+    if not gameDB:
+        gameDB = os.path.join(pyfaPath, "eve.db")
 
-    ## DON'T MODIFY ANYTHING BELOW ##
+    imgsZIP = getattr(configforced, "imgsZIP", imgsZIP)
+    if not imgsZIP:
+        imgsZIP = os.path.join(pyfaPath, "imgs.zip")
+
+    if debug:
+        logFile = "pyfa_debug.log"
+    else:
+        logFile = "pyfa.log"
+
+    logPath = os.path.join(savePath, logFile)
+
+    experimentalFeatures = getattr(configforced, "experimentalFeatures", experimentalFeatures)
+    if experimentalFeatures is None:
+        experimentalFeatures = False
+
+    # DON'T MODIFY ANYTHING BELOW
     import eos.config
 
-    #Caching modifiers, disable all gamedata caching, its unneeded.
+    # Caching modifiers, disable all gamedata caching, its unneeded.
     eos.config.gamedataCache = False
     # saveddata db location modifier, shouldn't ever need to touch this
     eos.config.saveddata_connectionstring = "sqlite:///" + saveDB + "?check_same_thread=False"
     eos.config.gamedata_connectionstring = "sqlite:///" + gameDB + "?check_same_thread=False"
+
+    # initialize the settings
+    from service.settings import EOSSettings, LocaleSettings
+    eos.config.settings = EOSSettings.getInstance().EOSSettings  # this is kind of confusing, but whatever
+
+    # set langauge, taking the passed argument or falling back to what's saved in the settings
+    localeSettings = LocaleSettings.getInstance()
+    language = language or localeSettings.get('locale')
+
+    # sets the lang for eos, using the mapped langauge.
+    eos.config.set_lang(localeSettings.get_eos_locale())
+
+def defLogging():
+    global debug
+    global logPath
+    global loggingLevel
+    global logging_setup
+
+    try:
+        if debug:
+            logging_setup = NestedSetup([
+                # make sure we never bubble up to the stderr handler
+                # if we run out of setup handling
+                NullHandler(),
+                StreamHandler(
+                        sys.stdout,
+                        bubble=False,
+                        level=loggingLevel
+                ),
+                TimedRotatingFileHandler(
+                        logPath,
+                        level=0,
+                        backup_count=3,
+                        bubble=True,
+                        date_format='%Y-%m-%d',
+                ),
+            ])
+        else:
+            logging_setup = NestedSetup([
+                # make sure we never bubble up to the stderr handler
+                # if we run out of setup handling
+                NullHandler(),
+                FingersCrossedHandler(
+                        TimedRotatingFileHandler(
+                                logPath,
+                                level=0,
+                                backup_count=3,
+                                bubble=False,
+                                date_format='%Y-%m-%d',
+                        ),
+                        action_level=ERROR,
+                        buffer_size=1000,
+                        # pull_information=True,
+                        # reset=False,
+                )
+            ])
+    except (KeyboardInterrupt, SystemExit):
+        raise
+    except:
+        print("Critical error attempting to setup logging. Falling back to console only.")
+        logging_setup = NestedSetup([
+            # make sure we never bubble up to the stderr handler
+            # if we run out of setup handling
+            NullHandler(),
+            StreamHandler(
+                    sys.stdout,
+                    bubble=False
+            )
+        ])
+
+
+class LoggerWriter:
+    def __init__(self, level):
+        # self.level is really like using log.debug(message)
+        # at least in my case
+        self.level = level
+
+    def write(self, message):
+        # if statement reduces the amount of newlines that are
+        # printed to the logger
+        if message.strip() != '':
+            self.level(message.replace("\n", ""))
+
+    def flush(self):
+        # create a flush method so things can be flushed when
+        # the system wants to. Not sure if simply 'printing'
+        # sys.stderr is the correct way to do it, but it seemed
+        # to work properly for me.
+        self.level(sys.stderr)

crowdin.yml

@@ -0,0 +1,3 @@
+files:
+  - source: /locale/*.pot
+    translation: /locale/%locale_with_underscore%/LC_MESSAGES/lang.po

db_update.py

@@ -0,0 +1,807 @@
+#!/usr/bin/env python3
+#======================================================================
+# Copyright (C) 2012 Diego Duclos
+#
+# This file is part of eos.
+#
+# eos is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as
+# published by the Free Software Foundation, either version 3 of
+# the License, or (at your option) any later version.
+#
+# eos is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with eos.  If not, see <http://www.gnu.org/licenses/>.
+#======================================================================
+
+
+import functools
+import itertools
+import json
+import os
+import re
+import sqlite3
+import sys
+
+import sqlalchemy.orm
+from sqlalchemy import or_, and_
+
+
+# todo: need to set the EOS language to en, becasuse this assumes it's being run within an English context
+# Need to know what that would do if called from pyfa
+ROOT_DIR = os.path.realpath(os.path.dirname(__file__))
+DB_PATH = os.path.join(ROOT_DIR, 'eve.db')
+JSON_DIR = os.path.join(ROOT_DIR, 'staticdata')
+if ROOT_DIR not in sys.path:
+    sys.path.insert(0, ROOT_DIR)
+GAMEDATA_SCHEMA_VERSION = 4
+
+
+def db_needs_update():
+    """True if needs, false if it does not, none if we cannot check it."""
+    try:
+        with open(os.path.join(JSON_DIR, 'phobos', 'metadata.0.json')) as f:
+            data_version = next((r['field_value'] for r in json.load(f) if r['field_name'] == 'client_build'))
+    except (KeyboardInterrupt, SystemExit):
+        raise
+    # If we have no source data - return None; should not update in this case
+    except:
+        return None
+    if not os.path.isfile(DB_PATH):
+        print('Gamedata DB not found')
+        return True
+    db_data_version = None
+    db_schema_version = None
+    try:
+        db = sqlite3.connect(DB_PATH)
+        cursor = db.cursor()
+        cursor.execute('SELECT field_value FROM metadata WHERE field_name = \'client_build\'')
+        for row in cursor:
+            db_data_version = int(row[0])
+        cursor.execute('SELECT field_value FROM metadata WHERE field_name = \'schema_version\'')
+        for row in cursor:
+            db_schema_version = int(row[0])
+        cursor.close()
+        db.close()
+    except (KeyboardInterrupt, SystemExit):
+        raise
+    except:
+        print('Error when fetching gamedata DB metadata')
+        return True
+    if data_version != db_data_version:
+        print('Gamedata DB data version mismatch: needed {}, DB has {}'.format(data_version, db_data_version))
+        return True
+    if GAMEDATA_SCHEMA_VERSION != db_schema_version:
+        print('Gamedata DB schema version mismatch: needed {}, DB has {}'.format(GAMEDATA_SCHEMA_VERSION, db_schema_version))
+        return True
+    return False
+
+
+def update_db():
+
+    print('Building gamedata DB...')
+
+    if os.path.isfile(DB_PATH):
+        os.remove(DB_PATH)
+
+    import eos.db
+    import eos.gamedata
+    import eos.config
+
+    # Create the database tables
+    eos.db.gamedata_meta.create_all()
+
+    def _readData(minerName, jsonName, keyIdName=None):
+        compiled_data = None
+        for i in itertools.count(0):
+            try:
+                with open(os.path.join(JSON_DIR, minerName, '{}.{}.json'.format(jsonName, i)), encoding='utf-8') as f:
+                    rawData = json.load(f)
+                    if i == 0:
+                        compiled_data = {} if type(rawData) == dict else []
+                    if type(rawData) == dict:
+                        compiled_data.update(rawData)
+                    else:
+                        compiled_data.extend(rawData)
+            except FileNotFoundError:
+                break
+
+        if not keyIdName:
+            return compiled_data
+        # IDs in keys, rows in values
+        data = []
+        for k, v in compiled_data.items():
+            row = {}
+            row.update(v)
+            row[keyIdName] = int(k)
+            data.append(row)
+        return data
+
+    def _addRows(data, cls, fieldMap=None):
+        if fieldMap is None:
+            fieldMap = {}
+        for row in data:
+            instance = cls()
+            for k, v in row.items():
+                if isinstance(v, str):
+                    v = v.strip()
+                setattr(instance, fieldMap.get(k, k), v)
+            eos.db.gamedata_session.add(instance)
+
+    def processEveTypes():
+        print('processing evetypes')
+        data = _readData('fsd_binary', 'types', keyIdName='typeID')
+        for row in data:
+            if (
+                # Apparently people really want Civilian modules available
+                (row['typeName_en-us'].startswith('Civilian') and "Shuttle" not in row['typeName_en-us'])
+                or row['typeName_en-us'] == 'Capsule'
+                or row['groupID'] == 4033  # destructible effect beacons
+                or re.match(r'AIR .+Booster.*', row['typeName_en-us'])
+            ):
+                row['published'] = True
+            # Nearly useless and clutter search results too much
+            elif (
+                row['typeName_en-us'].startswith('Limited Synth ')
+                or row['typeName_en-us'].startswith('Expired ')
+                or re.match(r'Mining Blitz .+ Booster Dose .+', row['typeName_en-us'])
+                or row['typeName_en-us'].endswith(' Filament') and (
+                    "'Needlejack'" not in row['typeName_en-us'] and
+                    "'Devana'" not in row['typeName_en-us'] and
+                    "'Pochven'" not in row['typeName_en-us'] and
+                    "'Extraction'" not in row['typeName_en-us'] and
+                    "'Krai Veles'" not in row['typeName_en-us'] and
+                    "'Krai Perun'" not in row['typeName_en-us'] and
+                    "'Krai Svarog'" not in row['typeName_en-us']
+                )
+            ):
+                row['published'] = False
+
+        newData = []
+        for row in data:
+            if (
+                row['published'] or
+                # group Ship Modifiers, for items like tactical t3 ship modes
+                row['groupID'] == 1306 or
+                # Micro Bombs (Fighters)
+                row['typeID'] in (41549, 41548, 41551, 41550) or
+                # Abyssal weather (environment)
+                row['groupID'] in (
+                    1882,
+                    1975,
+                    1971,
+                    1983)  # the "container" for the abyssal environments
+            ):
+                newData.append(row)
+        map = {'typeName_en-us': 'typeName', 'description_en-us': '_description'}
+        map.update({'description'+v: '_description'+v for (k, v) in eos.config.translation_mapping.items() if k != 'en'})
+        _addRows(newData, eos.gamedata.Item, fieldMap=map)
+        return newData
+
+    def processEveGroups():
+        print('processing evegroups')
+        data = _readData('fsd_binary', 'groups', keyIdName='groupID')
+        map = {'groupName_en-us': 'name'}
+        map.update({'groupName'+v: 'name'+v for (k, v) in eos.config.translation_mapping.items() if k != 'en'})
+        _addRows(data, eos.gamedata.Group, fieldMap=map)
+        return data
+
+    def processEveCategories():
+        print('processing evecategories')
+        data = _readData('fsd_binary', 'categories', keyIdName='categoryID')
+        map = { 'categoryName_en-us': 'name' }
+        map.update({'categoryName'+v: 'name'+v for (k, v) in eos.config.translation_mapping.items() if k != 'en'})
+        _addRows(data, eos.gamedata.Category, fieldMap=map)
+
+    def processDogmaAttributes():
+        print('processing dogmaattributes')
+        data = _readData('fsd_binary', 'dogmaattributes', keyIdName='attributeID')
+        map = {
+            'displayName_en-us': 'displayName',
+            # 'tooltipDescription_en-us': 'tooltipDescription'
+        }
+        _addRows(data, eos.gamedata.AttributeInfo, fieldMap=map)
+
+    def processDogmaTypeAttributes(eveTypesData):
+        print('processing dogmatypeattributes')
+        data = _readData('fsd_binary', 'typedogma', keyIdName='typeID')
+        eveTypeIds = set(r['typeID'] for r in eveTypesData)
+        newData = []
+        seenKeys = set()
+
+        def checkKey(key):
+            if key in seenKeys:
+                return False
+            seenKeys.add(key)
+            return True
+
+        for typeData in data:
+            if typeData['typeID'] not in eveTypeIds:
+                continue
+            for row in typeData.get('dogmaAttributes', ()):
+                row['typeID'] = typeData['typeID']
+                if checkKey((row['typeID'], row['attributeID'])):
+                    newData.append(row)
+        for row in eveTypesData:
+            for attrId, attrName in {4: 'mass', 38: 'capacity', 161: 'volume', 162: 'radius'}.items():
+                if attrName in row and checkKey((row['typeID'], attrId)):
+                    newData.append({'typeID': row['typeID'], 'attributeID': attrId, 'value': row[attrName]})
+
+        _addRows(newData, eos.gamedata.Attribute)
+        return newData
+
+    def processDynamicItemAttributes():
+        print('processing dynamicitemattributes')
+        data = _readData('fsd_binary', 'dynamicitemattributes')
+        for mutaID, mutaData in data.items():
+            muta = eos.gamedata.DynamicItem()
+            muta.typeID = mutaID
+            muta.resultingTypeID = mutaData['inputOutputMapping'][0]['resultingType']
+            eos.db.gamedata_session.add(muta)
+
+            for x in mutaData['inputOutputMapping'][0]['applicableTypes']:
+                item = eos.gamedata.DynamicItemItem()
+                item.typeID = mutaID
+                item.applicableTypeID = x
+                eos.db.gamedata_session.add(item)
+
+            for attrID, attrData in mutaData['attributeIDs'].items():
+                attr = eos.gamedata.DynamicItemAttribute()
+                attr.typeID = mutaID
+                attr.attributeID = attrID
+                attr.min = attrData['min']
+                attr.max = attrData['max']
+                eos.db.gamedata_session.add(attr)
+
+    def processDogmaEffects():
+        print('processing dogmaeffects')
+        data = _readData('fsd_binary', 'dogmaeffects', keyIdName='effectID')
+        _addRows(data, eos.gamedata.Effect, fieldMap={'resistanceAttributeID': 'resistanceID'})
+
+    def processDogmaTypeEffects(eveTypesData):
+        print('processing dogmatypeeffects')
+        data = _readData('fsd_binary', 'typedogma', keyIdName='typeID')
+        eveTypeIds = set(r['typeID'] for r in eveTypesData)
+        newData = []
+        for typeData in data:
+            if typeData['typeID'] not in eveTypeIds:
+                continue
+            for row in typeData.get('dogmaEffects', ()):
+                row['typeID'] = typeData['typeID']
+                newData.append(row)
+        _addRows(newData, eos.gamedata.ItemEffect)
+        return newData
+
+    def processDogmaUnits():
+        print('processing dogmaunits')
+        data = _readData('fsd_binary', 'dogmaunits', keyIdName='unitID')
+        _addRows(data, eos.gamedata.Unit, fieldMap={
+            'name': 'unitName',
+            'displayName_en-us': 'displayName'
+        })
+
+    def processMarketGroups():
+        print('processing marketgroups')
+        data = _readData('fsd_binary', 'marketgroups', keyIdName='marketGroupID')
+        map = {
+            'name_en-us': 'marketGroupName',
+            'description_en-us': '_description',
+        }
+        map.update({'name'+v: 'marketGroupName'+v for (k, v) in eos.config.translation_mapping.items() if k != 'en'})
+        map.update({'description' + v: '_description' + v for (k, v) in eos.config.translation_mapping.items() if k != 'en'})
+        _addRows(data, eos.gamedata.MarketGroup, fieldMap=map)
+
+    def processMetaGroups():
+        print('processing metagroups')
+        data = _readData('fsd_binary', 'metagroups', keyIdName='metaGroupID')
+        map = {'name_en-us': 'metaGroupName'}
+        map.update({'name' + v: 'metaGroupName' + v for (k, v) in eos.config.translation_mapping.items() if k != 'en'})
+        _addRows(data, eos.gamedata.MetaGroup, fieldMap=map)
+
+    def processCloneGrades():
+        print('processing clonegrades')
+        data = _readData('fsd_lite', 'clonegrades')
+
+        newData = []
+        # December, 2017 - CCP decided to use only one set of skill levels for alpha clones. However, this is still
+        # represented in the data as a skillset per race. To ensure that all skills are the same, we store them in a way
+        # that we can check to make sure all races have the same skills, as well as skill levels
+        check = {}
+        for ID in data:
+            for skill in data[ID]['skills']:
+                newData.append({
+                    'alphaCloneID': int(ID),
+                    'alphaCloneName': 'Alpha Clone',
+                    'typeID': skill['typeID'],
+                    'level': skill['level']})
+                if ID not in check:
+                    check[ID] = {}
+                check[ID][int(skill['typeID'])] = int(skill['level'])
+        if not functools.reduce(lambda a, b: a if a == b else False, [v for _, v in check.items()]):
+            raise Exception('Alpha Clones not all equal')
+        newData = [x for x in newData if x['alphaCloneID'] == 1]
+        if len(newData) == 0:
+            raise Exception('Alpha Clone processing failed')
+
+        tmp = []
+        for row in newData:
+            if row['alphaCloneID'] not in tmp:
+                cloneParent = eos.gamedata.AlphaClone()
+                setattr(cloneParent, 'alphaCloneID', row['alphaCloneID'])
+                setattr(cloneParent, 'alphaCloneName', row['alphaCloneName'])
+                eos.db.gamedata_session.add(cloneParent)
+                tmp.append(row['alphaCloneID'])
+        _addRows(newData, eos.gamedata.AlphaCloneSkill)
+
+    def processTraits():
+        print('processing traits')
+        data = _readData('phobos', 'traits')
+
+        def convertSection(sectionData):
+            sectionLines = []
+            headerText = '<b>{}</b>'.format(sectionData['header'])
+            sectionLines.append(headerText)
+            for bonusData in sectionData['bonuses']:
+                prefix = '{} '.format(bonusData['number']) if 'number' in bonusData else ''
+                bonusText = '{}{}'.format(prefix, bonusData['text'].replace('\u00B7', '\u2022 '))
+                sectionLines.append(bonusText)
+            sectionLine = '<br />\n'.join(sectionLines)
+            return sectionLine
+
+        newData = []
+        for row in data:
+            try:
+                newRow = {
+                    'typeID': row['typeID'],
+                }
+                for (k, v) in eos.config.translation_mapping.items():
+                    if v == '':
+                        v = '_en-us'
+                    typeLines = []
+                    traitData = row['traits{}'.format(v)]
+                    for skillData in sorted(traitData.get('skills', ()), key=lambda i: i['header']):
+                        typeLines.append(convertSection(skillData))
+                    if 'role' in traitData:
+                        typeLines.append(convertSection(traitData['role']))
+                    if 'misc' in traitData:
+                        typeLines.append(convertSection(traitData['misc']))
+                    traitLine = '<br />\n<br />\n'.join(typeLines)
+                    newRow['traitText{}'.format(v)] = traitLine
+
+                newData.append(newRow)
+            except:
+                pass
+        _addRows(newData, eos.gamedata.Traits, fieldMap={'traitText_en-us': 'traitText'})
+
+    def processMetadata():
+        print('processing metadata')
+        data = _readData('phobos', 'metadata')
+        _addRows(data, eos.gamedata.MetaData)
+
+    def processReqSkills(eveTypesData):
+        print('processing requiredskillsfortypes')
+
+        def composeReqSkills(raw):
+            reqSkills = {}
+            for skillTypeID, skillLevel in raw.items():
+                reqSkills[int(skillTypeID)] = skillLevel
+            return reqSkills
+
+        eveTypeIds = set(r['typeID'] for r in eveTypesData)
+        data = _readData('fsd_binary', 'requiredskillsfortypes')
+        reqsByItem = {}
+        itemsByReq = {}
+        for typeID, skillreqData in data.items():
+            typeID = int(typeID)
+            if typeID not in eveTypeIds:
+                continue
+            for skillTypeID, skillLevel in composeReqSkills(skillreqData).items():
+                reqsByItem.setdefault(typeID, {})[skillTypeID] = skillLevel
+                itemsByReq.setdefault(skillTypeID, {})[typeID] = skillLevel
+        for item in eos.db.gamedata_session.query(eos.gamedata.Item).all():
+            if item.typeID in reqsByItem:
+                item.reqskills = json.dumps(reqsByItem[item.typeID])
+            if item.typeID in itemsByReq:
+                item.requiredfor = json.dumps(itemsByReq[item.typeID])
+
+    def processReplacements(eveTypesData, eveGroupsData, dogmaTypeAttributesData, dogmaTypeEffectsData):
+        print('finding item replacements')
+
+        def compareAttrs(attrs1, attrs2):
+            # Consider items as different if they have no attrs
+            if len(attrs1) == 0 and len(attrs2) == 0:
+                return False
+            if set(attrs1) != set(attrs2):
+                return False
+            if all(attrs1[aid] == attrs2[aid] for aid in attrs1):
+                return True
+            return False
+
+        skillReqAttribs = {
+            182: 277,
+            183: 278,
+            184: 279,
+            1285: 1286,
+            1289: 1287,
+            1290: 1288}
+        skillReqAttribsFlat = set(skillReqAttribs.keys()).union(skillReqAttribs.values())
+        # Get data on type groups
+        # Format: {type ID: group ID}
+        typesGroups = {}
+        for row in eveTypesData:
+            typesGroups[row['typeID']] = row['groupID']
+        # Get data on item effects
+        # Format: {type ID: set(effect, IDs)}
+        typesEffects = {}
+        for row in dogmaTypeEffectsData:
+            typesEffects.setdefault(row['typeID'], set()).add(row['effectID'])
+        # Get data on type attributes
+        # Format: {type ID: {attribute ID: attribute value}}
+        typesNormalAttribs = {}
+        typesSkillAttribs = {}
+        for row in dogmaTypeAttributesData:
+            attributeID = row['attributeID']
+            if attributeID in skillReqAttribsFlat:
+                typeSkillAttribs = typesSkillAttribs.setdefault(row['typeID'], {})
+                typeSkillAttribs[row['attributeID']] = row['value']
+            # Ignore these attributes for comparison purposes
+            elif attributeID in (
+                # We do not need mass as it affects final ship stats only when carried by ship itself
+                # (and we're not going to replace ships), but it's wildly inconsistent for other items,
+                # which otherwise would be the same
+                4,  # mass
+                124,  # mainColor
+                162,  # radius
+                422,  # techLevel
+                633,  # metaLevel
+                1692,  # metaGroupID
+                1768  # typeColorScheme
+            ):
+                continue
+            else:
+                typeNormalAttribs = typesNormalAttribs.setdefault(row['typeID'], {})
+                typeNormalAttribs[row['attributeID']] = row['value']
+        # Get data on skill requirements
+        # Format: {type ID: {skill type ID: skill level}}
+        typesSkillReqs = {}
+        for typeID, typeAttribs in typesSkillAttribs.items():
+            typeSkillAttribs = typesSkillAttribs.get(typeID, {})
+            if not typeSkillAttribs:
+                continue
+            typeSkillReqs = typesSkillReqs.setdefault(typeID, {})
+            for skillreqTypeAttr, skillreqLevelAttr in skillReqAttribs.items():
+                try:
+                    skillType = int(typeSkillAttribs[skillreqTypeAttr])
+                    skillLevel = int(typeSkillAttribs[skillreqLevelAttr])
+                except (KeyError, ValueError):
+                    continue
+                typeSkillReqs[skillType] = skillLevel
+        # Format: {group ID: category ID}
+        groupCategories = {}
+        for row in eveGroupsData:
+            groupCategories[row['groupID']] = row['categoryID']
+        # As EVE affects various types mostly depending on their group or skill requirements,
+        # we're going to group various types up this way
+        # Format: {(group ID, frozenset(skillreq, type, IDs), frozenset(type, effect, IDs): [type ID, {attribute ID: attribute value}]}
+        groupedData = {}
+        for row in eveTypesData:
+            typeID = row['typeID']
+            # Ignore items outside of categories we need
+            if groupCategories[typesGroups[typeID]] not in (
+                6,  # Ship
+                7,  # Module
+                8,  # Charge
+                18,  # Drone
+                20,  # Implant
+                22,  # Deployable
+                23,  # Starbase
+                32,  # Subsystem
+                35,  # Decryptors
+                65,  # Structure
+                66,  # Structure Module
+                87,  # Fighter
+            ):
+                continue
+            typeAttribs = typesNormalAttribs.get(typeID, {})
+            # Ignore items w/o attributes
+            if not typeAttribs:
+                continue
+            # We need only skill types, not levels for keys
+            typeSkillreqs = frozenset(typesSkillReqs.get(typeID, {}))
+            typeGroup = typesGroups[typeID]
+            typeEffects = frozenset(typesEffects.get(typeID, ()))
+            groupData = groupedData.setdefault((typeGroup, typeSkillreqs, typeEffects), [])
+            groupData.append((typeID, typeAttribs))
+        # Format: {type ID: set(type IDs)}
+        replacements = {}
+        # Now, go through composed groups and for every item within it
+        # find items which are the same
+        for groupData in groupedData.values():
+            for type1, type2 in itertools.combinations(groupData, 2):
+                if compareAttrs(type1[1], type2[1]):
+                    replacements.setdefault(type1[0], set()).add(type2[0])
+                    replacements.setdefault(type2[0], set()).add(type1[0])
+        # Update DB session with data we generated
+        for item in eos.db.gamedata_session.query(eos.gamedata.Item).all():
+            itemReplacements = replacements.get(item.typeID)
+            if itemReplacements is not None:
+                item.replacements = ','.join('{}'.format(tid) for tid in sorted(itemReplacements))
+
+    def processImplantSets(eveTypesData):
+        print('composing implant sets')
+        # Includes only implants which can be considered part of sets, not all implants
+        implant_groups = (300, 1730)
+        specials = {'Genolution': ('Genolution Core Augmentation', r'CA-\d+')}
+        implantSets = {}
+        for row in eveTypesData:
+            if not row.get('published'):
+                continue
+            if row.get('groupID') not in implant_groups:
+                continue
+            typeName = row.get('typeName_en-us', '')
+            # Regular sets matching
+            m = re.match(r'(?P<grade>(High|Mid|Low)-grade) (?P<set>\w+) (?P<implant>(Alpha|Beta|Gamma|Delta|Epsilon|Omega))', typeName, re.IGNORECASE)
+            if m:
+                implantSets.setdefault((m.group('grade'), m.group('set')), set()).add(row['typeID'])
+            # Special set matching
+            for setHandle, (setName, implantPattern) in specials.items():
+                pattern = '(?P<set>{}) (?P<implant>{})'.format(setName, implantPattern)
+                m = re.match(pattern, typeName)
+                if m:
+                    implantSets.setdefault((None, setHandle), set()).add(row['typeID'])
+                    break
+        data = []
+        for (gradeName, setName), implants in implantSets.items():
+            if len(implants) < 2:
+                continue
+            implants = ','.join('{}'.format(tid) for tid in sorted(implants))
+            row = {'setName': setName, 'gradeName': gradeName, 'implants': implants}
+            data.append(row)
+        _addRows(data, eos.gamedata.ImplantSet)
+
+    eveTypesData = processEveTypes()
+    eveGroupsData = processEveGroups()
+    processEveCategories()
+    processDogmaAttributes()
+    dogmaTypeAttributesData = processDogmaTypeAttributes(eveTypesData)
+    processDynamicItemAttributes()
+    processDogmaEffects()
+    dogmaTypeEffectsData = processDogmaTypeEffects(eveTypesData)
+    processDogmaUnits()
+    processMarketGroups()
+    processMetaGroups()
+    processCloneGrades()
+    processTraits()
+    processMetadata()
+
+    eos.db.gamedata_session.flush()
+    processReqSkills(eveTypesData)
+    processReplacements(eveTypesData, eveGroupsData, dogmaTypeAttributesData, dogmaTypeEffectsData)
+    processImplantSets(eveTypesData)
+
+    # Add schema version to prevent further updates
+    metadata_schema_version = eos.gamedata.MetaData()
+    metadata_schema_version.field_name = 'schema_version'
+    metadata_schema_version.field_value = GAMEDATA_SCHEMA_VERSION
+    eos.db.gamedata_session.add(metadata_schema_version)
+
+    eos.db.gamedata_session.flush()
+
+    # CCP still has 5 subsystems assigned to T3Cs, even though only 4 are available / usable. They probably have some
+    # old legacy requirement or assumption that makes it difficult for them to change this value in the data. But for
+    # pyfa, we can do it here as a post-processing step
+    for attr in eos.db.gamedata_session.query(eos.gamedata.Attribute).filter(eos.gamedata.Attribute.ID == 1367).all():
+        attr.value = 4.0
+    for item in eos.db.gamedata_session.query(eos.gamedata.Item).filter(or_(
+        eos.gamedata.Item.name.like('%abyssal%'),
+        eos.gamedata.Item.name.like('%mutated%'),
+        eos.gamedata.Item.name.like('%_PLACEHOLDER%'),
+        # Drifter weapons are published for some reason
+        eos.gamedata.Item.name.in_(('Lux Kontos', 'Lux Xiphos'))
+    )).all():
+        if 'Asteroid Mining Crystal' in item.name:
+            continue
+        if 'Mutated Drone Specialization' in item.name:
+            continue
+        item.published = False
+
+    for x in [
+        30  # Apparel
+    ]:
+        cat = eos.db.gamedata_session.query(eos.gamedata.Category).filter(eos.gamedata.Category.ID == x).first()
+        print ('Removing Category: {}'.format(cat.name))
+        eos.db.gamedata_session.delete(cat)
+
+    # Unused normally, can be useful for customizing items
+    def _copyItem(srcName, tgtTypeID, tgtName):
+        eveType = eos.db.gamedata_session.query(eos.gamedata.Item).filter(eos.gamedata.Item.name == srcName).one()
+        eos.db.gamedata_session.expunge(eveType)
+        sqlalchemy.orm.make_transient(eveType)
+        eveType.ID = tgtTypeID
+        for suffix in eos.config.translation_mapping.values():
+            setattr(eveType, f'typeName{suffix}', tgtName)
+        eos.db.gamedata_session.add(eveType)
+        eos.db.gamedata_session.flush()
+
+    def _hardcodeAttribs(typeID, attrMap):
+        for attrName, value in attrMap.items():
+            try:
+                attr = eos.db.gamedata_session.query(eos.gamedata.Attribute).filter(and_(
+                    eos.gamedata.Attribute.name == attrName, eos.gamedata.Attribute.typeID == typeID)).one()
+            except sqlalchemy.orm.exc.NoResultFound:
+                attrInfo = eos.db.gamedata_session.query(eos.gamedata.AttributeInfo).filter(eos.gamedata.AttributeInfo.name == attrName).one()
+                attr = eos.gamedata.Attribute()
+                attr.attributeID = attrInfo.ID
+                attr.typeID = typeID
+                attr.value = value
+                eos.db.gamedata_session.add(attr)
+            else:
+                attr.value = value
+
+    def _hardcodeEffects(typeID, effectMap):
+        item = eos.db.gamedata_session.query(eos.gamedata.Item).filter(eos.gamedata.Item.ID == typeID).one()
+        item.effects.clear()
+        for effectID, effectName in effectMap.items():
+            effect = eos.gamedata.Effect()
+            effect.effectID = effectID
+            effect.effectName = effectName
+            item.effects[effectName] = effect
+
+    def hardcodeShapash():
+        shapashTypeID = 1000000
+        _copyItem(srcName='Utu', tgtTypeID=shapashTypeID, tgtName='Shapash')
+        attrMap = {
+            # Fitting
+            'powerOutput': 50,
+            'cpuOutput': 225,
+            'capacitorCapacity': 420,
+            'rechargeRate': 187500,
+            # Slots
+            'hiSlots': 3,
+            'medSlots': 4,
+            'lowSlots': 4,
+            'launcherSlotsLeft': 0,
+            'turretSlotsLeft': 3,
+            # Rigs
+            'rigSlots': 2,
+            'rigSize': 1,
+            'upgradeCapacity': 400,
+            # Shield
+            'shieldCapacity': 575,
+            'shieldRechargeRate': 625000,
+            'shieldEmDamageResonance': 1 - 0.0,
+            'shieldThermalDamageResonance': 1 - 0.6,
+            'shieldKineticDamageResonance': 1 - 0.85,
+            'shieldExplosiveDamageResonance': 1 - 0.5,
+            # Armor
+            'armorHP': 1015,
+            'armorEmDamageResonance': 1 - 0.5,
+            'armorThermalDamageResonance': 1 - 0.675,
+            'armorKineticDamageResonance': 1 - 0.8375,
+            'armorExplosiveDamageResonance': 1 - 0.1,
+            # Structure
+            'hp': 1274,
+            'emDamageResonance': 1 - 0.33,
+            'thermalDamageResonance': 1 - 0.33,
+            'kineticDamageResonance': 1 - 0.33,
+            'explosiveDamageResonance': 1 - 0.33,
+            'mass': 1215000,
+            'volume': 29500,
+            'capacity': 165,
+            # Navigation
+            'maxVelocity': 325,
+            'agility': 3.467,
+            'warpSpeedMultiplier': 5.5,
+            # Drones
+            'droneCapacity': 75,
+            'droneBandwidth': 25,
+            # Targeting
+            'maxTargetRange': 49000,
+            'maxLockedTargets': 6,
+            'scanRadarStrength': 0,
+            'scanLadarStrength': 0,
+            'scanMagnetometricStrength': 9,
+            'scanGravimetricStrength': 0,
+            'signatureRadius': 39,
+            'scanResolution': 550,
+            # Misc
+            'energyWarfareResistance': 0,
+            'stasisWebifierResistance': 0,
+            'weaponDisruptionResistance': 0}
+        effectMap = {
+            100100: 'pyfaCustomShapashAfArAmount',
+            100101: 'pyfaCustomShapashAfShtTrackingOptimal',
+            100102: 'pyfaCustomShapashGfShtDamage',
+            100103: 'pyfaCustomShapashGfPointRange',
+            100104: 'pyfaCustomShapashGfPropOverheat',
+            100105: 'pyfaCustomShapashRolePlateMass',
+            100106: 'pyfaCustomShapashRoleHeat'}
+        _hardcodeAttribs(shapashTypeID, attrMap)
+        _hardcodeEffects(shapashTypeID, effectMap)
+
+    def hardcodeCybele():
+        cybeleTypeID = 1000001
+        _copyItem(srcName='Adrestia', tgtTypeID=cybeleTypeID, tgtName='Cybele')
+        attrMap = {
+            # Fitting
+            'powerOutput': 1284,
+            'cpuOutput': 400,
+            'capacitorCapacity': 2400,
+            'rechargeRate': 334000,
+            'hiSlots': 5,
+            'medSlots': 4,
+            'lowSlots': 6,
+            'launcherSlotsLeft': 0,
+            'turretSlotsLeft': 5,
+            # Rigs
+            'rigSlots': 2,
+            'rigSize': 2,
+            'upgradeCapacity': 400,
+            # Shield
+            'shieldCapacity': 1200,
+            'shieldRechargeRate': 1250000,
+            'shieldEmDamageResonance': 1 - 0.0,
+            'shieldThermalDamageResonance': 1 - 0.5,
+            'shieldKineticDamageResonance': 1 - 0.9,
+            'shieldExplosiveDamageResonance': 1 - 0.5,
+            # Armor
+            'armorHP': 1900,
+            'armorEmDamageResonance': 1 - 0.5,
+            'armorThermalDamageResonance': 1 - 0.69,
+            'armorKineticDamageResonance': 1 - 0.85,
+            'armorExplosiveDamageResonance': 1 - 0.1,
+            # Structure
+            'hp': 2300,
+            'emDamageResonance': 1 - 0.33,
+            'thermalDamageResonance': 1 - 0.33,
+            'kineticDamageResonance': 1 - 0.33,
+            'explosiveDamageResonance': 1 - 0.33,
+            'mass': 11100000,
+            'volume': 112000,
+            'capacity': 450,
+            # Navigation
+            'maxVelocity': 235,
+            'agility': 0.457,
+            'warpSpeedMultiplier': 4.5,
+            # Drones
+            'droneCapacity': 100,
+            'droneBandwidth': 50,
+            # Targeting
+            'maxTargetRange': 60000,
+            'maxLockedTargets': 6,
+            'scanRadarStrength': 0,
+            'scanLadarStrength': 0,
+            'scanMagnetometricStrength': 15,
+            'scanGravimetricStrength': 0,
+            'signatureRadius': 115,
+            'scanResolution': 330,
+            # Misc
+            'energyWarfareResistance': 0,
+            'stasisWebifierResistance': 0,
+            'weaponDisruptionResistance': 0}
+        effectMap = {
+            100200: 'pyfaCustomCybeleHacMhtFalloff',
+            100201: 'pyfaCustomCybeleHacMhtTracking',
+            100202: 'pyfaCustomCybeleGcMhtDamage',
+            100203: 'pyfaCustomCybeleGcArAmount',
+            100204: 'pyfaCustomCybeleGcPointRange',
+            100205: 'pyfaCustomCybeleRoleVelocity',
+            100206: 'pyfaCustomCybeleRolePlateMass'}
+        _hardcodeAttribs(cybeleTypeID, attrMap)
+        _hardcodeEffects(cybeleTypeID, effectMap)
+
+    # hardcodeShapash()
+    # hardcodeCybele()
+
+    eos.db.gamedata_session.commit()
+    eos.db.gamedata_engine.execute('VACUUM')
+
+    print('done')
+
+
+if __name__ == '__main__':
+    update_db()

dist_assets/linux/AppImageBuilder.yml

@@ -0,0 +1,74 @@
+version: 1
+
+AppDir:
+  path: ./AppDir
+
+  app_info:
+    id: pyfa
+    name: pyfa
+    icon: pyfa
+    version: '{{PYFA_VERSION}}'
+    exec: usr/bin/python3.11
+    exec_args: "-s $APPDIR/opt/pyfa/pyfa.py $@"
+
+  apt:
+    arch: [ amd64 ]
+    sources:
+    - sourceline: 'deb http://us.archive.ubuntu.com/ubuntu jammy main restricted universe multiverse'
+      key_url: 'http://keyserver.ubuntu.com/pks/lookup?op=get&search=0x871920d1991bc93c'
+    - sourceline: 'deb http://us.archive.ubuntu.com/ubuntu jammy-updates main restricted universe multiverse'
+      key_url: 'http://keyserver.ubuntu.com/pks/lookup?op=get&search=0x871920d1991bc93c'
+    - sourceline: 'deb http://us.archive.ubuntu.com/ubuntu jammy-backports main restricted universe multiverse'
+      key_url: 'http://keyserver.ubuntu.com/pks/lookup?op=get&search=0x871920d1991bc93c'
+    - sourceline: 'deb http://us.archive.ubuntu.com/ubuntu jammy-security main restricted universe multiverse'
+      key_url: 'http://keyserver.ubuntu.com/pks/lookup?op=get&search=0x871920d1991bc93c'
+    - sourceline: 'deb https://ppa.launchpadcontent.net/deadsnakes/ppa/ubuntu jammy main'
+      key_url: 'http://keyserver.ubuntu.com/pks/lookup?op=get&search=0xf23c5a6cf475977595c89f51ba6932366a755776'
+
+    include:
+      - python3.11
+      # wx dependencies
+      - libgtk-3-0
+      - librsvg2-common  # GTK3 recommendation; without it, search in char editor crashes
+      - libwebkit2gtk-4.0-37  # Needed for wx's HTML lib
+      # Unknown
+      - libpcre2-32-0  # https://github.com/pyfa-org/Pyfa/issues/2572
+      - libnotify4  # https://github.com/pyfa-org/Pyfa/issues/2598
+      - libwayland-client0  # https://github.com/pyfa-org/Pyfa/issues/2600
+    exclude:
+      - hicolor-icon-theme
+      - humanity-icon-theme
+      - ubuntu-mono
+
+  after_bundle:
+    # Install python dependencies to bundled interpreter
+    - export PYTHONHOME="AppDir/usr"
+    - export PYTHONPATH="AppDir/usr/lib/python3.11/site-packages"
+    - curl -L https://bootstrap.pypa.io/get-pip.py -o get-pip.py
+    - AppDir/usr/bin/python3.11 get-pip.py
+    # Just to bundle certificates with AppImage
+    - AppDir/usr/bin/python3.11 -s -m pip install certifi
+    - AppDir/usr/bin/python3.11 -s -m pip install -f https://extras.wxpython.org/wxPython4/extras/linux/gtk3/ubuntu-22.04 -r requirements.txt
+
+  files:
+    exclude:
+      - usr/lib/x86_64-linux-gnu/gconv
+      - usr/share/man
+      - usr/share/doc/*/README.*
+      - usr/share/doc/*/changelog.*
+      - usr/share/doc/*/NEWS.*
+      - usr/share/doc/*/TODO.*
+      - usr/include
+
+  runtime:
+    env:
+      PYTHONHOME: '${APPDIR}/usr'
+      PYTHONPATH: '${APPDIR}/usr/lib/python3.11/site-packages'
+      SSL_CERT_FILE: '${APPDIR}/usr/local/lib/python3.11/dist-packages/certifi/cacert.pem'
+      # Workaround for https://github.com/AppImageCrafters/appimage-builder/issues/336
+      XDG_DATA_DIRS: '${APPDIR}/usr/local/share:${APPDIR}/usr/share:/usr/local/share:/usr/share:$XDG_DATA_DIRS'
+
+AppImage:
+  sign-key: None
+  arch: x86_64
+  file_name: 'pyfa-{{PYFA_VERSION}}-linux.AppImage'

dist_assets/mac/pyfa.spec

@@ -0,0 +1,95 @@
+# -*- mode: python -*-
+
+import os
+from itertools import chain
+import subprocess
+import requests.certs
+
+label = subprocess.check_output([
+    "git", "describe", "--tags"]).strip()
+
+with open('.version', 'w+') as f:
+    f.write(label.decode())
+
+block_cipher = None
+
+added_files = [
+             ('../../imgs/gui/*.png', 'imgs/gui'),
+             ('../../imgs/gui/*.gif', 'imgs/gui'),
+             ('../../imgs/icons/*.png', 'imgs/icons'),
+             ('../../imgs/renders/*.png', 'imgs/renders'),
+             ('../../dist_assets/win/pyfa.ico', '.'),
+             ('../../service/jargon/*.yaml', 'service/jargon'),
+             ('../../locale', 'locale'),
+             (requests.certs.where(), '.'),  # is this needed anymore?
+             ('../../eve.db', '.'),
+             ('../../README.md', '.'),
+             ('../../LICENSE', '.'),
+             ('../../version.yml', '.'),
+             ]
+
+
+import_these = [
+    'numpy.core._dtype_ctypes',  # https://github.com/pyinstaller/pyinstaller/issues/3982
+    'sqlalchemy.ext.baked',  # windows build doesn't launch without if when using sqlalchemy 1.3.x
+    'pkg_resources.py2_warn'  # issue 2156
+]
+
+icon = os.path.join(os.getcwd(), "dist_assets", "mac", "pyfa.icns")
+
+# Walk directories that do dynamic importing
+paths = ('eos/db/migrations', 'service/conversions')
+for root, folders, files in chain.from_iterable(os.walk(path) for path in paths):
+    for file_ in files:
+        if file_.endswith(".py") and not file_.startswith("_"):
+            mod_name = "{}.{}".format(
+                root.replace("/", "."),
+                file_.split(".py")[0],
+            )
+            import_these.append(mod_name)
+
+a = Analysis([r'../../pyfa.py'],
+             pathex=[],
+             binaries=[],
+             datas=added_files,
+             hiddenimports=import_these,
+             hookspath=['dist_assets/pyinstaller_hooks'],
+             runtime_hooks=[],
+             excludes=[],
+             win_no_prefer_redirects=False,
+             win_private_assemblies=False,
+             cipher=block_cipher)
+
+pyz = PYZ(a.pure, a.zipped_data,
+             cipher=block_cipher)
+
+exe = EXE(pyz,
+          a.scripts,
+          a.binaries,
+          a.zipfiles,
+          a.datas,
+          name='pyfa',
+          debug=False,
+          strip=False,
+          upx=True,
+          runtime_tmpdir=None,
+          console=False ,
+          icon=icon,
+          )
+
+app = BUNDLE(
+    exe,
+    name='pyfa.app',
+    version=os.getenv('PYFA_VERSION'),
+    icon=icon,
+    bundle_identifier=None,
+    info_plist={
+        'NSHighResolutionCapable': 'True',
+        'NSPrincipalClass': 'NSApplication',
+        'CFBundleName': 'pyfa',
+        'CFBundleDisplayName': 'pyfa',
+        'CFBundleIdentifier': 'org.pyfaorg.pyfa',
+        'CFBundleVersion': os.getenv('PYFA_VERSION'),
+        'CFBundleShortVersionString': os.getenv('PYFA_VERSION'),
+    }
+)

dist_assets/win/dist.py

@@ -0,0 +1,37 @@
+# helper script to zip up pyinstaller distribution and create installer file
+
+import os.path
+from subprocess import call
+import zipfile
+from packaging.version import Version
+import yaml
+
+
+with open("version.yml", 'r') as file:
+    data = yaml.load(file, Loader=yaml.SafeLoader)
+    version = data['version']
+
+os.environ["PYFA_DIST_DIR"] = os.path.join(os.getcwd(), 'dist')
+
+os.environ["PYFA_VERSION"] = version
+iscc = r"C:\Program Files (x86)\Inno Setup 6\ISCC.exe"
+
+source = os.path.join(os.environ["PYFA_DIST_DIR"], "pyfa")
+
+fileName = "pyfa-{}-win".format(os.environ["PYFA_VERSION"])
+
+print("Compiling EXE")
+
+v = Version(version)
+
+print(v)
+
+call([
+    iscc,
+    os.path.join(os.getcwd(), "dist_assets", "win", "pyfa-setup.iss"),
+    "/dMyAppVersion=%s" % v,
+    "/dMyAppDir=%s" % source,
+    "/dMyOutputDir=%s" % os.path.join(os.getcwd()),
+    "/dMyOutputFile=%s" % fileName])  # stdout=devnull, stderr=devnull
+
+print("Done")

dist_assets/win/pyfa-setup.iss

@@ -0,0 +1,187 @@
+; Script generated by the Inno Setup Script Wizard.
+; SEE THE DOCUMENTATION FOR DETAILS ON CREATING INNO SETUP SCRIPT FILES!
+
+; Versioning
+; we do some #ifdef conditionals because automated compilation passes these as arguments
+
+#ifndef MyAppVersion
+    #define MyAppVersion "2.1.0"
+#endif
+
+; Other config
+
+#define MyAppName "pyfa"
+#define MyAppPublisher "pyfa"
+#define MyAppURL "https://github.com/pyfa-org/Pyfa/"
+#define MyAppExeName "pyfa.exe"
+
+#ifndef MyOutputFile
+    #define MyOutputFile LowerCase(StringChange(MyAppName+'-'+MyAppVersion+'-win', " ", "-"))
+#endif
+#ifndef MyAppDir
+    #define MyAppDir "pyfa"
+#endif
+#ifndef MyOutputDir
+    #define MyOutputDir "dist"
+#endif
+
+[Setup]
+; NOTE: The value of AppId uniquely identifies this application.
+; Do not use the same AppId value in installers for other applications.
+; (To generate a new GUID, click Tools | Generate GUID inside the IDE.)
+AppId={{3DA39096-C08D-49CD-90E0-1D177F32C8AA}
+AppName={#MyAppName}
+AppVersion={#MyAppVersion}
+AppPublisher={#MyAppPublisher}
+AppPublisherURL={#MyAppURL}
+AppSupportURL={#MyAppURL}
+AppUpdatesURL={#MyAppURL}
+ArchitecturesAllowed=x64
+ArchitecturesInstallIn64BitMode=x64
+CloseApplications=yes
+DefaultDirName={pf}\{#MyAppName}
+DefaultGroupName={#MyAppName}
+AllowNoIcons=yes
+LicenseFile={#MyAppDir}\app\LICENSE
+OutputDir={#MyOutputDir}
+OutputBaseFilename={#MyOutputFile}
+SetupIconFile={#MyAppDir}\app\pyfa.ico
+SolidCompression=yes
+
+[Languages]
+Name: "english"; MessagesFile: "compiler:Default.isl"
+
+[Tasks]
+Name: "desktopicon"; Description: "{cm:CreateDesktopIcon}"; GroupDescription: "{cm:AdditionalIcons}"; Flags: unchecked
+Name: "quicklaunchicon"; Description: "{cm:CreateQuickLaunchIcon}"; GroupDescription: "{cm:AdditionalIcons}"; Flags: unchecked; OnlyBelowVersion: 0,6.1
+
+[Files]
+Source: "{#MyAppDir}\pyfa.exe"; DestDir: "{app}"; Flags: ignoreversion; AfterInstall: RemoveFromVirtualStore
+Source: "{#MyAppDir}\*"; DestDir: "{app}"; Flags: ignoreversion recursesubdirs createallsubdirs
+; NOTE: Don't use "Flags: ignoreversion" on any shared system files
+
+[Icons]
+Name: "{group}\{#MyAppName}"; Filename: "{app}\{#MyAppExeName}"
+Name: "{group}\{cm:UninstallProgram,{#MyAppName}}"; Filename: "{uninstallexe}"
+Name: "{commondesktop}\{#MyAppName}"; Filename: "{app}\{#MyAppExeName}"; Tasks: desktopicon
+Name: "{userappdata}\Microsoft\Internet Explorer\Quick Launch\{#MyAppName}"; Filename: "{app}\{#MyAppExeName}"; Tasks: quicklaunchicon
+
+[Run]
+Filename: "{app}\{#MyAppExeName}"; Description: "{cm:LaunchProgram,{#StringChange(MyAppName, '&', '&&')}}"; Flags: nowait postinstall skipifsilent
+
+[InstallDelete]
+; These will delete left over generated files from 1.14 and below
+Type: filesandordirs; Name: "{app}\eos"
+Type: filesandordirs; Name: "{app}\gui"
+Type: filesandordirs; Name: "{app}\service"
+Type: filesandordirs; Name: "{app}\utils"
+Type: files; Name: "{app}\*.pyo"
+Type: files; Name: "{app}\*.pyc"
+
+[Code]
+
+/////////////////////////////////////////////////////////////////////
+function IsAppRunning(const FileName : string): Boolean;
+var
+    FSWbemLocator: Variant;
+    FWMIService   : Variant;
+    FWbemObjectSet: Variant;
+begin
+    Result := false;
+    FSWbemLocator := CreateOleObject('WBEMScripting.SWBEMLocator');
+    FWMIService := FSWbemLocator.ConnectServer('', 'root\CIMV2', '', '');
+    FWbemObjectSet := FWMIService.ExecQuery(Format('SELECT Name FROM Win32_Process Where Name="%s"',[FileName]));
+    Result := (FWbemObjectSet.Count > 0);
+    FWbemObjectSet := Unassigned;
+    FWMIService := Unassigned;
+    FSWbemLocator := Unassigned;
+end;
+
+/////////////////////////////////////////////////////////////////////
+procedure RemoveFromVirtualStore;
+var
+    VirtualStore,FileName,FilePath:String;
+    DriveChars:Integer;
+begin
+    VirtualStore:=AddBackslash(ExpandConstant('{localappdata}'))+'VirtualStore';
+    FileName:=ExpandConstant(CurrentFileName);
+    DriveChars:=Length(ExtractFileDrive(FileName));
+    if DriveChars>0 then begin
+        Delete(FileName,1,DriveChars);
+        FileName:=VirtualStore+FileName;
+        FilePath:=ExtractFilePath(FileName);
+        DelTree(FilePath, True, True, True);
+    end;
+end;
+
+/////////////////////////////////////////////////////////////////////
+function PrepareToInstall(var NeedsRestart: Boolean): String;
+begin
+  if(IsAppRunning( 'pyfa.exe' )) then
+    begin
+      Result := 'Please close pyfa before continuing. When closed, please go back to the previous step and continue. If you have named this installer pyfa.exe, please rename it and restart installation';
+    end
+  else
+    begin
+      Result := '';
+    end
+end;
+
+/////////////////////////////////////////////////////////////////////
+function GetUninstallString(): String;
+var
+  sUnInstPath: String;
+  sUnInstallString: String;
+begin
+  sUnInstPath := ExpandConstant('Software\Microsoft\Windows\CurrentVersion\Uninstall\{{3DA39096-C08D-49CD-90E0-1D177F32C8AA}_is1'); //Your App GUID/ID
+  sUnInstallString := '';
+  if not RegQueryStringValue(HKLM, sUnInstPath, 'UninstallString', sUnInstallString) then
+    if not RegQueryStringValue(HKCU, sUnInstPath, 'UninstallString', sUnInstallString) then
+      if not RegQueryStringValue(HKLM32, sUnInstPath, 'UninstallString', sUnInstallString) then
+        RegQueryStringValue(HKCU32, sUnInstPath, 'UninstallString', sUnInstallString);
+  Result := sUnInstallString;
+end;
+
+/////////////////////////////////////////////////////////////////////
+function UnInstallOldVersion(): Integer;
+var
+  sUnInstallString: String;
+  iResultCode: Integer;
+begin
+// Return Values:
+// 1 - uninstall string is empty
+// 2 - error executing the UnInstallString
+// 3 - successfully executed the UnInstallString
+
+  // default return value
+  Result := 0;
+
+  // get the uninstall string of the old app
+  sUnInstallString := GetUninstallString();
+  if sUnInstallString <> '' then begin
+    sUnInstallString := RemoveQuotes(sUnInstallString);
+    if Exec(sUnInstallString, '/SILENT /NORESTART /SUPPRESSMSGBOXES','', SW_HIDE, ewWaitUntilTerminated, iResultCode) then
+      Result := 3
+    else
+      Result := 2;
+  end else
+    Result := 1;
+end;
+
+/////////////////////////////////////////////////////////////////////
+function IsUpgrade(): Boolean;
+begin
+  Result := (GetUninstallString() <> '');
+end;
+
+/////////////////////////////////////////////////////////////////////
+procedure CurStepChanged(CurStep: TSetupStep);
+begin
+  if (CurStep=ssInstall) then
+  begin
+    if (IsUpgrade()) then
+    begin
+      UnInstallOldVersion();
+    end;
+  end;
+end;

dist_assets/win/pyfa.exe.manifest

@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<assembly xmlns="urn:schemas-microsoft-com:asm.v1" manifestVersion="1.0">
+  <assemblyIdentity name="pyfa" processorArchitecture="x86" type="win32" version="1.0.0.0"/>
+  <trustInfo xmlns="urn:schemas-microsoft-com:asm.v3">
+    <security>
+      <requestedPrivileges>
+        <requestedExecutionLevel level="asInvoker" uiAccess="false"></requestedExecutionLevel>
+      </requestedPrivileges>
+    </security>
+  </trustInfo>
+  <compatibility xmlns="urn:schemas-microsoft-com:compatibility.v1">
+    <application>
+      <!-- Windows Vista -->
+      <supportedOS Id="{e2011457-1546-43c5-a5fe-008deee3d3f0}"/>
+      <!-- Windows 7 -->
+      <supportedOS Id="{35138b9a-5d96-4fbd-8e2d-a2440225f93a}"/>
+      <!-- Windows 8 -->
+      <supportedOS Id="{4a2f28e3-53b9-4441-ba9c-d69d4a4a6e38}"/>
+      <!-- Windows 8.1 -->
+      <supportedOS Id="{1f676c76-80e1-4239-95bb-83d0f6d0da78}"/>
+      <!-- Windows 10 and Windows 11 -->
+      <supportedOS Id="{8e0f7a12-bfb3-4fe8-b9a5-48fd50a15a9a}"/>
+    </application>
+  </compatibility>
+</assembly>

dist_assets/win/pyfa.spec

@@ -0,0 +1,87 @@
+# -*- mode: python -*-
+
+import os
+from itertools import chain
+import subprocess
+import requests.certs
+
+label = subprocess.check_output(["git", "describe", "--tags"]).strip()
+
+with open('.version', 'w+') as f:
+    f.write(label.decode())
+
+block_cipher = None
+
+added_files = [
+             ('../../imgs/gui/*.png', 'imgs/gui'),
+             ('../../imgs/gui/*.gif', 'imgs/gui'),
+             ('../../imgs/icons/*.png', 'imgs/icons'),
+             ('../../imgs/renders/*.png', 'imgs/renders'),
+             ('../../service/jargon/*.yaml', 'service/jargon'),
+             ('../../dist_assets/win/pyfa.ico', '.'),
+             ('../../dist_assets/win/pyfa.exe.manifest', '.'),
+             (requests.certs.where(), '.'),  # is this needed anymore?
+             ('../../eve.db', '.'),
+             ('../../README.md', '.'),
+             ('../../LICENSE', '.'),
+             ('../../version.yml', '.'),
+             ]
+
+import_these = [
+    'numpy.core._dtype_ctypes',  # https://github.com/pyinstaller/pyinstaller/issues/3982
+    'sqlalchemy.ext.baked',  # windows build doesn't launch without if when using sqlalchemy 1.3.x
+    'pkg_resources.py2_warn'  # issue 2156
+]
+
+# Walk directories that do dynamic importing
+paths = ('eos/db/migrations', 'service/conversions')
+for root, folders, files in chain.from_iterable(os.walk(path) for path in paths):
+    for file_ in files:
+        if file_.endswith(".py") and not file_.startswith("_"):
+            mod_name = "{}.{}".format(
+                root.replace("/", "."),
+                file_.split(".py")[0],
+            )
+            import_these.append(mod_name)
+
+a = Analysis(['../../pyfa.py'],
+             pathex=[
+                 # Need this, see https://github.com/pyinstaller/pyinstaller/issues/1566
+                 # To get this, download and install windows 10 SDK
+                 # If not building on Windows 10, this might be optional
+                 r'C:\Program Files (x86)\Windows Kits\10\Redist\ucrt\DLLs\x86'],
+             binaries=[],
+             datas=added_files,
+             hiddenimports=import_these,
+             hookspath=['dist_assets/pyinstaller_hooks'],
+             runtime_hooks=[],
+             excludes=['Tkinter'],
+             win_no_prefer_redirects=False,
+             win_private_assemblies=False,
+             cipher=block_cipher)
+
+pyz = PYZ(a.pure, a.zipped_data,
+             cipher=block_cipher)
+
+exe = EXE(pyz,
+          a.scripts,
+          exclude_binaries=True,
+          debug=False,
+          console=False,
+          strip=False,
+          upx=True,
+          name='pyfa',
+          icon='dist_assets/win/pyfa.ico',
+          )
+
+coll = COLLECT(
+               exe,
+               a.binaries,
+               a.zipfiles,
+               a.datas,
+               strip=False,
+               upx=True,
+               name='pyfa',
+               icon='dist_assets/win/pyfa.ico',
+               )
+

dist_assets/win/version_resource.py

@@ -0,0 +1,45 @@
+# UTF-8
+#
+# For more details about fixed file info 'ffi' see:
+# http://msdn.microsoft.com/en-us/library/ms646997.aspx
+VSVersionInfo(
+  ffi=FixedFileInfo(
+    # filevers and prodvers should be always a tuple with four items: (1, 2, 3, 4)
+    # Set not needed items to zero 0.
+    filevers=(1, 15, 1, 0),
+    prodvers=(1, 15, 1, 0),
+    # Contains a bitmask that specifies the valid bits 'flags'r
+    mask=0x3f,
+    # Contains a bitmask that specifies the Boolean attributes of the file.
+    flags=0x0,
+    # The operating system for which this file was designed.
+    # 0x4 - NT and there is no need to change it.
+    OS=0x40004,
+    # The general type of file.
+    # 0x1 - the file is an application.
+    fileType=0x1,
+    # The function of the file.
+    # 0x0 - the function is not defined for this fileType
+    subtype=0x0,
+    # Creation date and time stamp.
+    date=(0, 0)
+    ),
+  kids=[
+    StringFileInfo(
+      [
+      StringTable(
+        u'040904E4',
+        [StringStruct(u'LegalCopyright', u''),
+        StringStruct(u'InternalName', u'pyfa.exe'),
+        StringStruct(u'FileVersion', u'1.15.1.0'),
+        StringStruct(u'CompanyName', u''),
+        StringStruct(u'OriginalFilename', u'pyfa.exe'),
+        StringStruct(u'ProductVersion', u'1.15.1.0'),
+        StringStruct(u'FileDescription', u'Python fitting assistant'),
+        StringStruct(u'LegalTrademarks', u''),
+        StringStruct(u'Comments', u''),
+        StringStruct(u'ProductName', u'pyfa')])
+      ]),
+    VarFileInfo([VarStruct(u'Translation', [1033, 1252])])
+  ]
+)

docs/_config.yml

@@ -0,0 +1 @@
+theme: jekyll-theme-midnight

docs/callback.html

@@ -0,0 +1,104 @@
+<html>
+  <head>
+    <meta http-equiv="Content-type" content="text/html;charset=UTF-8" />
+    <title>pyfa Authentication Proxy</title>
+    <style type="text/css">
+
+    
+      body { width: 700px; margin: 150px auto; }
+      h1 { font-size: 40px; }
+      h2 { font-size: 32px; }
+      body { font: 20px Helvetica, sans-serif; color: #333; }
+      #article { display: block; text-align: left; width: 650px; margin: 0 auto; }
+      .hidden { display:none; }
+      .success { color: #28a745; }
+      .error { color: #dc3545; }
+      textarea { width: 100%; height: 100px; }
+      hr { border-width: 1px 0 0 0; border-style: solid; border-color: #333; }
+      button { cursor: pointer; background-color: white; border: 1px solid #333; color: #333; padding: 8px 11px; text-align: center; text-decoration: none; display: inline-block; }
+      @media (prefers-color-scheme: dark) {
+          body { background-color: #333; color: white; }
+          textarea { width: 100%; height: 100px; background-color: #aaa;}
+          button { background-color: #333; border: 1px solid white; color: white;  }
+          hr { border-color: white; }
+      }
+    </style>
+  </head>
+
+  <body>
+    <!-- Layout from Short Circuit's CREST login. Shout out! https://github.com/farshield/shortcircuit -->
+      <h1>pyfa</h1>
+      <div id="mode0" class="hidden">
+        <p id="mode0-msg">Processing request...</p>
+        <button id="showBtn" onClick="showCode()">Show Code</button>
+      </div>
+      <div id="mode1" class="hidden">
+        <p id="mode1-p">Please copy and paste the token below into pyfa.</p>
+        <textarea id="authCodeText" readonly></textarea>
+      </div>
+      
+    </div>
+    <script>
+      debugger;
+      function showCode() {
+        var element = document.getElementById(`mode1`);
+        element.classList.remove("hidden");
+        element = document.getElementById(`showBtn`);
+        element.classList.add("hidden"); 
+      }
+
+      function httpPostAsync(url, callback)
+      {
+          var xmlHttp = new XMLHttpRequest();
+          xmlHttp.onreadystatechange = function() { 
+              if (xmlHttp.readyState == XMLHttpRequest.DONE)
+                  callback(xmlHttp);
+          }
+          xmlHttp.open("GET", url, true); // true for asynchronous 
+          xmlHttp.send(null);
+      }
+
+      const urlSearchParams = new URLSearchParams(window.location.search);
+      const params = Object.fromEntries(urlSearchParams.entries());
+      let stateInfo;
+      try {
+        stateInfo = JSON.parse(atob(params.state))
+      } catch (err) {
+        // something has happened and we cannot continue.
+        // todo: show a simple message and exit.
+        throw err;
+      }
+
+      // we always want to show the text box for manual.
+      var element = document.getElementById(`mode${stateInfo.mode}`);
+      element.classList.remove("hidden");
+
+      if (stateInfo.mode === 0) {
+        let p = document.getElementById(`mode1-p`);
+        p.prepend(document.createElement("hr"))
+      }
+      debugger;
+      document.getElementById(`authCodeText`).value = btoa(JSON.stringify(params))
+
+      if (stateInfo.mode == 0) { // auto / server mode 
+        httpPostAsync(stateInfo.redirect + window.location.search, (req)=>{
+          debugger;
+          const msgDiv = document.getElementById(`mode0-msg`);
+          if (req.status === 200) {
+            msgDiv.innerHTML ="<span class='success'>Success!</span> You may close this window and return to the application.";
+            return;
+          } else if (req.status === 0){
+            msgDiv.innerHTML = "<span class='error'>Error!</span> Server response not received.<p><small>The local pyfa server may have timed out, or may not have started correctly.</small></p>";
+          } else if (req.status === 400){
+            msgDiv.innerHTML = `<span class='error'>Error!</span> <p><small>${req.responseText}</small></p>`
+          } else {
+            msgDiv.innerHTML = `<span class='error'>Error!</span> <p><small>There was an unknown error. Please report this to the pyfa issues page.</p><p><textarea readdonly>${req.responseText}</textarea></small></p>`
+          }
+          showCode()
+          // todo: bad request error when it's not  an error itself, but rather
+        })
+        // todo: post message to local EVE server
+      }
+    </script>
+  </body>
+</html>

docs/index.md

@@ -0,0 +1,37 @@
+## Welcome to GitHub Pages
+
+You can use the [editor on GitHub](https://github.com/pyfa-org/Pyfa/edit/gh-pages/index.md) to maintain and preview the content for your website in Markdown files.
+
+Whenever you commit to this repository, GitHub Pages will run [Jekyll](https://jekyllrb.com/) to rebuild the pages in your site, from the content in your Markdown files.
+
+### Markdown
+
+Markdown is a lightweight and easy-to-use syntax for styling your writing. It includes conventions for
+
+```markdown
+Syntax highlighted code block
+
+# Header 1
+## Header 2
+### Header 3
+
+- Bulleted
+- List
+
+1. Numbered
+2. List
+
+**Bold** and _Italic_ and `Code` text
+
+[Link](url) and ![Image](src)
+```
+
+For more details see [GitHub Flavored Markdown](https://guides.github.com/features/mastering-markdown/).
+
+### Jekyll Themes
+
+Your Pages site will use the layout and styles from the Jekyll theme you have selected in your [repository settings](https://github.com/pyfa-org/Pyfa/settings/pages). The name of this theme is saved in the Jekyll `_config.yml` configuration file.
+
+### Support or Contact
+
+Having trouble with Pages? Check out our [documentation](https://docs.github.com/categories/github-pages-basics/) or [contact support](https://support.github.com/contact) and we’ll help you sort it out.

eos/__init__.py

@@ -1,8 +1,2 @@
-version = "0.2.3"
-tag = "git"
-
-
-def test():
-    import tests.runTests
-    import unittest
-    unittest.main(defaultTest="discover", testLoader=tests.runTests.loader)
+version = "0.2.3"
+tag = "git"

eos/calc.py

@@ -0,0 +1,71 @@
+# =============================================================================
+# Copyright (C) 2019 Ryan Holmes
+#
+# This file is part of pyfa.
+#
+# pyfa is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# pyfa is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with pyfa.  If not, see <http://www.gnu.org/licenses/>.
+# =============================================================================
+
+
+import math
+
+
+# Just copy-paste penalization chain calculation code (with some modifications,
+# as multipliers arrive in different form) in here to not make actual attribute
+# calculations slower than they already are due to extra function calls
+def calculateMultiplier(multipliers):
+    """
+    multipliers: dictionary in format:
+    {stacking group name: [(mult, resist attr ID), (mult, resist attr ID)]}
+    """
+    val = 1
+    for penalizedMultipliers in multipliers.values():
+        # A quick explanation of how this works:
+        # 1: Bonuses and penalties are calculated seperately, so we'll have to filter each of them
+        l1 = [v[0] for v in penalizedMultipliers if v[0] > 1]
+        l2 = [v[0] for v in penalizedMultipliers if v[0] < 1]
+        # 2: The most significant bonuses take the smallest penalty,
+        # This means we'll have to sort
+        abssort = lambda _val: -abs(_val - 1)
+        l1.sort(key=abssort)
+        l2.sort(key=abssort)
+        # 3: The first module doesn't get penalized at all
+        # Any module after the first takes penalties according to:
+        # 1 + (multiplier - 1) * math.exp(- math.pow(i, 2) / 7.1289)
+        for l in (l1, l2):
+            for i in range(len(l)):
+                bonus = l[i]
+                val *= 1 + (bonus - 1) * math.exp(- i ** 2 / 7.1289)
+    return val
+
+
+def calculateRangeFactor(srcOptimalRange, srcFalloffRange, distance, restrictedRange=True):
+    """Range strength/chance factor, applicable to guns, ewar, RRs, etc."""
+    if distance is None:
+        return 1
+    if srcFalloffRange > 0:
+        # Most modules cannot be activated when at 3x falloff range, with few exceptions like guns
+        if restrictedRange and distance > srcOptimalRange + 3 * srcFalloffRange:
+            return 0
+        return 0.5 ** ((max(0, distance - srcOptimalRange) / srcFalloffRange) ** 2)
+    elif distance <= srcOptimalRange:
+        return 1
+    else:
+        return 0
+
+
+def calculateLockTime(srcScanRes, tgtSigRadius):
+    if not srcScanRes or not tgtSigRadius:
+        return None
+    return min(40000 / srcScanRes / math.asinh(tgtSigRadius) ** 2, 30 * 60)

eos/capSim.py

@@ -1,6 +1,7 @@
 import heapq
 import time
 from math import sqrt, exp
+from collections import Counter
 
 DAY = 24 * 60 * 60 * 1000
 
@@ -12,7 +13,7 @@ def lcm(a, b):
     return n / a
 
 
-class CapSimulator(object):
+class CapSimulator:
     """Entity's EVE Capacitor Simulator"""
 
     def __init__(self):
@@ -20,6 +21,7 @@ class CapSimulator(object):
 
         self.capacitorCapacity = 100
         self.capacitorRecharge = 1000
+        self.startingCapacity = 1000
 
         # max simulated time.
         self.t_max = DAY
@@ -40,6 +42,14 @@ class CapSimulator(object):
         # relevant decimal digits of capacitor for LCM period optimization
         self.stability_precision = 1
 
+        # Stores how cap sim changed cap values outside of cap regen time
+        self.saved_changes = ()
+        self.saved_changes_internal = None
+
+        # Reports if sim was stopped due to detecting stability early
+        self.optimize_repeats = True
+        self.result_optimized_repeats = None
+
     def scale_activation(self, duration, capNeed):
         for res in self.scale_resolutions:
             mod = duration % res
@@ -58,7 +68,7 @@ class CapSimulator(object):
         return duration, capNeed
 
     def init(self, modules):
-        """prepare modules. a list of (duration, capNeed, clipSize, disableStagger) tuples is
+        """prepare modules. a list of (duration, capNeed, clipSize, disableStagger, reloadTime, isInjector) tuples is
          expected, with clipSize 0 if the module has infinite ammo.
             """
         self.modules = modules
@@ -66,47 +76,57 @@ class CapSimulator(object):
     def reset(self):
         """Reset the simulator state"""
         self.state = []
+        self.saved_changes_internal = {}
+        self.result_optimized_repeats = False
         mods = {}
         period = 1
         disable_period = False
 
         # Loop over modules, clearing clipSize if applicable, and group modules based on attributes
-        for (duration, capNeed, clipSize, disableStagger) in self.modules:
+        for (duration, capNeed, clipSize, disableStagger, reloadTime, isInjector) in self.modules:
             if self.scale:
                 duration, capNeed = self.scale_activation(duration, capNeed)
 
             # set clipSize to infinite if reloads are disabled unless it's
-            # a cap booster module.
-            if not self.reload and capNeed > 0:
+            # a cap booster module
+            if not self.reload and not isInjector:
                 clipSize = 0
+                reloadTime = 0
 
             # Group modules based on their properties
-            if (duration, capNeed, clipSize, disableStagger) in mods:
-                mods[(duration, capNeed, clipSize, disableStagger)] += 1
+            key = (duration, capNeed, clipSize, disableStagger, reloadTime, isInjector)
+            if key in mods:
+                mods[key] += 1
             else:
-                mods[(duration, capNeed, clipSize, disableStagger)] = 1
+                mods[key] = 1
 
         # Loop over grouped modules, configure staggering and push to the simulation state
-        for (duration, capNeed, clipSize, disableStagger), amount in mods.iteritems():
+        for (duration, capNeed, clipSize, disableStagger, reloadTime, isInjector), amount in mods.items():
+            # period optimization doesn't work when reloads are active.
+            if clipSize:
+                disable_period = True
+            # Just push multiple instances if item is injector. We do not want to stagger them as we will
+            # use them as needed and want them to be available right away
+            if isInjector:
+                for i in range(amount):
+                    heapq.heappush(self.state, [0, duration, capNeed, 0, clipSize, reloadTime, isInjector])
+                continue
             if self.stagger and not disableStagger:
+                # Stagger all mods if they do not need to be reloaded
                 if clipSize == 0:
                     duration = int(duration / amount)
+                # Stagger mods after first
                 else:
-                    stagger_amount = (duration * clipSize + 10000) / (amount * clipSize)
+                    stagger_amount = (duration * clipSize + reloadTime) / (amount * clipSize)
                     for i in range(1, amount):
-                        heapq.heappush(self.state,
-                                       [i * stagger_amount, duration,
-                                        capNeed, 0, clipSize])
+                        heapq.heappush(self.state, [i * stagger_amount, duration, capNeed, 0, clipSize, reloadTime, isInjector])
+            # If mods are not staggered - just multiply cap use
             else:
                 capNeed *= amount
 
             period = lcm(period, duration)
 
-            # period optimization doesn't work when reloads are active.
-            if clipSize:
-                disable_period = True
-
-            heapq.heappush(self.state, [0, duration, capNeed, 0, clipSize])
+            heapq.heappush(self.state, [0, duration, capNeed, 0, clipSize, reloadTime, isInjector])
 
         if disable_period:
             self.period = self.t_max
@@ -117,7 +137,8 @@ class CapSimulator(object):
         """Run the simulation"""
 
         start = time.time()
-
+        awaitingInjectors = []
+        awaitingInjectorsCounterWrap = Counter()
         self.reset()
 
         push = heapq.heappush
@@ -127,27 +148,36 @@ class CapSimulator(object):
         stability_precision = self.stability_precision
         period = self.period
 
+        activation = None
         iterations = 0
 
         capCapacity = self.capacitorCapacity
         tau = self.capacitorRecharge / 5.0
 
-        cap_wrap = capCapacity  # cap value at last period
-        cap_lowest = capCapacity  # lowest cap value encountered
-        cap_lowest_pre = capCapacity  # lowest cap value before activations
-        cap = capCapacity  # current cap value
+        cap_wrap = self.startingCapacity  # cap value at last period
+        cap_lowest = self.startingCapacity  # lowest cap value encountered
+        cap_lowest_pre = self.startingCapacity  # lowest cap value before activations
+        cap = self.startingCapacity  # current cap value
         t_wrap = self.period  # point in time of next period
-
         t_last = 0
         t_max = self.t_max
 
         while 1:
-            activation = pop(state)
-            t_now, duration, capNeed, shot, clipSize = activation
+            # Nothing to pop - might happen when no mods are activated, or when
+            # only cap injectors are active (and are postponed by code below)
+            try:
+                activation = pop(state)
+            except IndexError:
+                break
+            t_now, duration, capNeed, shot, clipSize, reloadTime, isInjector = activation
+
+            # Max time reached, stop simulation - we're stable
             if t_now >= t_max:
                 break
 
-            cap = ((1.0 + (sqrt(cap / capCapacity) - 1.0) * exp((t_last - t_now) / tau)) ** 2) * capCapacity
+            # Regenerate cap from last time point
+            if t_now > t_last:
+                cap = ((1.0 + (sqrt(cap / capCapacity) - 1.0) * exp((t_last - t_now) / tau)) ** 2) * capCapacity
 
             if t_now != t_last:
                 if cap < cap_lowest_pre:
@@ -155,36 +185,104 @@ class CapSimulator(object):
                 if t_now == t_wrap:
                     # history is repeating itself, so if we have more cap now than last
                     # time this happened, it is a stable setup.
-                    if cap >= cap_wrap:
+                    awaitingInjectorsCounterNow = Counter(awaitingInjectors)
+                    if self.optimize_repeats and cap >= cap_wrap and awaitingInjectorsCounterNow == awaitingInjectorsCounterWrap:
+                        self.result_optimized_repeats = True
                         break
                     cap_wrap = round(cap, stability_precision)
+                    awaitingInjectorsCounterWrap = awaitingInjectorsCounterNow
                     t_wrap += period
 
-            cap -= capNeed
-            if cap > capCapacity:
-                cap = capCapacity
-
+            t_last = t_now
             iterations += 1
 
-            t_last = t_now
+            # If injecting cap will "overshoot" max cap, postpone it
+            if isInjector and cap - capNeed > capCapacity:
+                awaitingInjectors.append((duration, capNeed, shot, clipSize, reloadTime, isInjector))
 
-            if cap < cap_lowest:
-                if cap < 0.0:
-                    break
-                cap_lowest = cap
+            else:
+                # If we will need more cap than we have, but we are not at 100% -
+                # use awaiting cap injectors to top us up until we have enough or
+                # until we're full
+                if capNeed > cap and cap < capCapacity:
+                    while awaitingInjectors and capNeed > cap and capCapacity > cap:
+                        neededInjection = min(capNeed - cap, capCapacity - cap)
+                        # Find injectors which have just enough cap or more
+                        goodInjectors = [i for i in awaitingInjectors if -i[1] >= neededInjection]
+                        if goodInjectors:
+                            # Pick injector which overshoots the least
+                            bestInjector = min(goodInjectors, key=lambda i: -i[1])
+                        else:
+                            # Take the one which provides the most cap
+                            bestInjector = max(goodInjectors, key=lambda i: -i[1])
+                        # Use injector
+                        awaitingInjectors.remove(bestInjector)
+                        inj_duration, inj_capNeed, inj_shot, inj_clipSize, inj_reloadTime, inj_isInjector = bestInjector
+                        cap -= inj_capNeed
+                        if cap > capCapacity:
+                            cap = capCapacity
+                        self.saved_changes_internal[t_now] = cap
+                        # Add injector to regular state tracker
+                        inj_t_now = t_now
+                        inj_t_now += inj_duration
+                        inj_shot += 1
+                        if inj_clipSize:
+                            if inj_shot % inj_clipSize == 0:
+                                inj_shot = 0
+                                inj_t_now += inj_reloadTime
+                        push(state, [inj_t_now, inj_duration, inj_capNeed, inj_shot, inj_clipSize, inj_reloadTime, inj_isInjector])
 
-            # queue the next activation of this module
-            t_now += duration
-            shot += 1
-            if clipSize:
-                if shot % clipSize == 0:
-                    shot = 0
-                    t_now += 10000  # include reload time
-            activation[0] = t_now
-            activation[3] = shot
+                # Apply cap modification
+                cap -= capNeed
+                if cap > capCapacity:
+                    cap = capCapacity
+                self.saved_changes_internal[t_now] = cap
 
+                if cap < cap_lowest:
+                    # Negative cap - we're unstable, simulation is over
+                    if cap < 0.0:
+                        break
+                    cap_lowest = cap
+
+                # Try using awaiting injectors to top up the cap after spending some
+                while awaitingInjectors and cap < capCapacity:
+                    neededInjection = capCapacity - cap
+                    # Find injectors which do not overshoot max cap
+                    goodInjectors = [i for i in awaitingInjectors if -i[1] <= neededInjection]
+                    if not goodInjectors:
+                        break
+                    # Take the one which provides the most cap
+                    bestInjector = max(goodInjectors, key=lambda i: -i[1])
+                    # Use injector
+                    awaitingInjectors.remove(bestInjector)
+                    inj_duration, inj_capNeed, inj_shot, inj_clipSize, inj_reloadTime, inj_isInjector = bestInjector
+                    cap -= inj_capNeed
+                    if cap > capCapacity:
+                        cap = capCapacity
+                    self.saved_changes_internal[t_now] = cap
+                    # Add injector to regular state tracker
+                    inj_t_now = t_now
+                    inj_t_now += inj_duration
+                    inj_shot += 1
+                    if inj_clipSize:
+                        if inj_shot % inj_clipSize == 0:
+                            inj_shot = 0
+                            inj_t_now += inj_reloadTime
+                    push(state, [inj_t_now, inj_duration, inj_capNeed, inj_shot, inj_clipSize, inj_reloadTime, inj_isInjector])
+
+                # queue the next activation of this module
+                t_now += duration
+                shot += 1
+                if clipSize:
+                    if shot % clipSize == 0:
+                        shot = 0
+                        t_now += reloadTime  # include reload time
+                activation[0] = t_now
+                activation[3] = shot
+
+                push(state, activation)
+        if activation is not None:
             push(state, activation)
-        push(state, activation)
 
         # update instance with relevant results.
         self.t = t_last
@@ -192,7 +290,7 @@ class CapSimulator(object):
 
         # calculate EVE's stability value
         try:
-            avgDrain = reduce(float.__add__, map(lambda x: x[2] / x[1], self.state), 0.0)
+            avgDrain = sum(x[2] / x[1] for x in self.state)
             self.cap_stable_eve = 0.25 * (1.0 + sqrt(-(2.0 * avgDrain * tau - capCapacity) / capCapacity)) ** 2
         except ValueError:
             self.cap_stable_eve = 0.0
@@ -202,7 +300,9 @@ class CapSimulator(object):
             self.cap_stable_low = cap_lowest
             self.cap_stable_high = cap_lowest_pre
         else:
-            self.cap_stable_low = \
-                self.cap_stable_high = 0.0
+            self.cap_stable_low = self.cap_stable_high = 0.0
+
+        self.saved_changes = tuple((k / 1000, max(0, self.saved_changes_internal[k])) for k in sorted(self.saved_changes_internal))
+        self.saved_changes_internal = None
 
         self.runtime = time.time() - start

eos/config.py

@@ -1,13 +1,51 @@
 import sys
 from os.path import realpath, join, dirname, abspath
 
+from logbook import Logger
+import os
+
+istravis = os.environ.get('TRAVIS') == 'true'
+pyfalog = Logger(__name__)
+
 debug = False
 gamedataCache = True
 saveddataCache = True
-gamedata_connectionstring = 'sqlite:///' + unicode(realpath(join(dirname(abspath(__file__)), "..", "eve.db")),
-                                                   sys.getfilesystemencoding())
-saveddata_connectionstring = 'sqlite:///' + unicode(
-    realpath(join(dirname(abspath(__file__)), "..", "saveddata", "saveddata.db")), sys.getfilesystemencoding())
+gamedata_version = ""
+gamedata_date = ""
+gamedata_connectionstring = 'sqlite:///' + realpath(join(dirname(abspath(__file__)), "..", "eve.db"))
+
+lang = ""
+
+# Maps supported langauges to their suffix in the database
+translation_mapping = {
+    "en": "",
+    "fr": "_fr",
+    # "it": "_it",
+    "ja": "_ja",
+    "ko": "_ko",
+    "ru": "_ru",
+    "zh": "_zh",
+}
+
+def set_lang(i18n_lang):
+    global lang
+    lang = translation_mapping.get(i18n_lang, translation_mapping.get("en"))
+
+pyfalog.debug("Gamedata connection string: {0}", gamedata_connectionstring)
+
+if istravis is True or hasattr(sys, '_called_from_test'):
+    # Running in Travis. Run saveddata database in memory.
+    saveddata_connectionstring = 'sqlite:///:memory:'
+else:
+    saveddata_connectionstring = 'sqlite:///' + realpath(join(dirname(abspath(__file__)), "..", "saveddata", "saveddata.db"))
+
+pyfalog.debug("Saveddata connection string: {0}", saveddata_connectionstring)
+
+settings = {
+    "useStaticAdaptiveArmorHardener": False,
+    "strictSkillLevels": True,
+    "globalDefaultSpoolupPercentage": 1.0
+}
 
 # Autodetect path, only change if the autodetection bugs out.
-path = dirname(unicode(__file__, sys.getfilesystemencoding()))
+path = dirname(__file__)

eos/const.py

@@ -0,0 +1,115 @@
+# =============================================================================
+# Copyright (C) 2019 Ryan Holmes
+#
+# This file is part of pyfa.
+#
+# pyfa is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# pyfa is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with pyfa.  If not, see <http://www.gnu.org/licenses/>.
+# =============================================================================
+
+from enum import IntEnum,unique
+
+
+@unique
+class FittingSlot(IntEnum):
+    """
+    Contains slots for ship fittings
+    """
+    # These are self-explanatory
+    LOW = 1
+    MED = 2
+    HIGH = 3
+    RIG = 4
+    SUBSYSTEM = 5
+    # not a real slot, need for pyfa display rack separation
+    MODE = 6
+    # system effects. They are projected "modules" and pyfa assumes all modules
+    # have a slot. In this case, make one up.
+    SYSTEM = 7
+    # used for citadel services
+    SERVICE = 8
+    # fighter 'slots'. Just easier to put them here...
+    F_LIGHT = 10
+    F_SUPPORT = 11
+    F_HEAVY = 12
+    # fighter 'slots' (for structures)
+    FS_LIGHT = 13
+    FS_SUPPORT = 14
+    FS_HEAVY = 15
+
+
+@unique
+class ImplantLocation(IntEnum):
+    """
+    Contains location of the implant
+    """
+    FIT = 0
+    CHARACTER = 1
+
+
+@unique
+class CalcType(IntEnum):
+    """
+    Contains location of the calculation
+    """
+    LOCAL = 0
+    PROJECTED = 1
+    COMMAND = 2
+
+
+@unique
+class FittingModuleState(IntEnum):
+    """
+    Contains the state of a fitting module
+    """
+    OFFLINE = -1
+    ONLINE = 0
+    ACTIVE = 1
+    OVERHEATED = 2
+
+
+@unique
+class FittingHardpoint(IntEnum):
+    """
+    Contains the types of a fitting hardpoint
+    """
+    NONE = 0
+    MISSILE = 1
+    TURRET = 2
+
+
+@unique
+class SpoolType(IntEnum):
+    # Spool and cycle scale are different in case if max spool amount cannot
+    # be divided by spool step without remainder
+    SPOOL_SCALE = 0  # [0..1]
+    CYCLE_SCALE = 1  # [0..1]
+    TIME = 2  # Expressed via time in seconds since spool up started
+    CYCLES = 3  # Expressed in amount of cycles since spool up started
+
+
+@unique
+class FitSystemSecurity(IntEnum):
+    HISEC = 0
+    LOWSEC = 1
+    NULLSEC = 2
+    WSPACE = 3
+
+
+@unique
+class Operator(IntEnum):
+    PREASSIGN = 0
+    PREINCREASE = 1
+    MULTIPLY = 2
+    POSTINCREASE = 3
+    FORCE = 4

eos/db/__init__.py

@@ -17,41 +17,84 @@
 # along with eos.  If not, see <http://www.gnu.org/licenses/>.
 # ===============================================================================
 
+import re
 import threading
 
-from sqlalchemy import MetaData, create_engine
+from sqlalchemy import MetaData, create_engine, event
 from sqlalchemy.orm import sessionmaker, scoped_session
-from sqlalchemy.ext.declarative import declarative_base
-from sqlalchemy import pool
 
-
-import migration
+from . import migration
 from eos import config
+from logbook import Logger
+
+
+pyfalog = Logger(__name__)
+pyfalog.info("Initializing database")
+pyfalog.info("Gamedata connection: {0}", config.gamedata_connectionstring)
+pyfalog.info("Saveddata connection: {0}", config.saveddata_connectionstring)
 
 
 class ReadOnlyException(Exception):
     pass
 
 
+def re_fn(expr, item):
+    try:
+        reg = re.compile(expr, re.IGNORECASE)
+    except (SystemExit, KeyboardInterrupt):
+        raise
+    except:
+        return False
+    return reg.search(item) is not None
+
+
+pyfalog.debug('Initializing gamedata')
 gamedata_connectionstring = config.gamedata_connectionstring
 if callable(gamedata_connectionstring):
     gamedata_engine = create_engine("sqlite://", creator=gamedata_connectionstring, echo=config.debug)
 else:
     gamedata_engine = create_engine(gamedata_connectionstring, echo=config.debug)
 
+
+@event.listens_for(gamedata_engine, 'connect')
+def create_functions(dbapi_connection, connection_record):
+    dbapi_connection.create_function('regexp', 2, re_fn)
+
+
 gamedata_meta = MetaData()
 gamedata_meta.bind = gamedata_engine
-gamedata_session = sessionmaker(bind=gamedata_engine, autoflush=False, expire_on_commit=False)()
+GamedataSession = scoped_session(sessionmaker(bind=gamedata_engine, autoflush=False, expire_on_commit=False))
+gamedata_session = GamedataSession()
 
+gamedata_sessions = {threading.get_ident(): gamedata_session}
+
+
+def get_gamedata_session():
+    thread_id = threading.get_ident()
+    if thread_id not in gamedata_sessions:
+        gamedata_sessions[thread_id] = GamedataSession()
+    return gamedata_sessions[thread_id]
+
+
+pyfalog.debug('Getting gamedata version')
 # This should be moved elsewhere, maybe as an actual query. Current, without try-except, it breaks when making a new
 # game db because we haven't reached gamedata_meta.create_all()
 try:
     config.gamedata_version = gamedata_session.execute(
-        "SELECT `field_value` FROM `metadata` WHERE `field_name` LIKE 'client_build'"
+            "SELECT `field_value` FROM `metadata` WHERE `field_name` LIKE 'client_build'"
     ).fetchone()[0]
-except:
+    config.gamedata_date = gamedata_session.execute(
+        "SELECT `field_value` FROM `metadata` WHERE `field_name` LIKE 'dump_time'"
+    ).fetchone()[0]
+except (KeyboardInterrupt, SystemExit):
+    raise
+except Exception as e:
+    pyfalog.warning("Missing gamedata version.")
+    pyfalog.critical(e)
     config.gamedata_version = None
+    config.gamedata_date = None
 
+pyfalog.debug('Initializing saveddata')
 saveddata_connectionstring = config.saveddata_connectionstring
 if saveddata_connectionstring is not None:
     if callable(saveddata_connectionstring):
@@ -62,23 +105,35 @@ if saveddata_connectionstring is not None:
     saveddata_meta = MetaData()
     saveddata_meta.bind = saveddata_engine
     saveddata_session = sessionmaker(bind=saveddata_engine, autoflush=False, expire_on_commit=False)()
+else:
+    saveddata_meta = None
 
 # Lock controlling any changes introduced to session
-sd_lock = threading.Lock()
+sd_lock = threading.RLock()
 
+pyfalog.debug('Importing gamedata DB scheme')
 # Import all the definitions for all our database stuff
-from eos.db.gamedata import *
-from eos.db.saveddata import *
+# noinspection PyPep8
+from eos.db.gamedata import alphaClones, attribute, category, effect, group, item, marketGroup, metaData, metaGroup, queries, traits, unit, dynamicAttributes, implantSet
+pyfalog.debug('Importing saveddata DB scheme')
+# noinspection PyPep8
+from eos.db.saveddata import booster, cargo, character, damagePattern, databaseRepair, drone, fighter, fit, implant, implantSet, \
+    miscData, mutatorMod, mutatorDrone, module, override, price, queries, skill, targetProfile, user
 
-# Import queries
+pyfalog.debug('Importing gamedata queries')
+# noinspection PyPep8
 from eos.db.gamedata.queries import *
+pyfalog.debug('Importing saveddata queries')
+# noinspection PyPep8
 from eos.db.saveddata.queries import *
 
 # If using in memory saveddata, you'll want to reflect it so the data structure is good.
 if config.saveddata_connectionstring == "sqlite:///:memory:":
     saveddata_meta.create_all()
+    pyfalog.info("Running database out of memory.")
 
 
 def rollback():
     with sd_lock:
+        pyfalog.warning("Session rollback triggered.")
         saveddata_session.rollback()

eos/db/gamedata/__init__.py

@@ -1,2 +1,2 @@
-__all__ = ["attribute", "category", "effect", "group", "metaData",
-           "icon", "item", "marketGroup", "metaGroup", "unit"]
+__all__ = ["attribute", "category", "effect", "group", "metaData", "dynamicAttributes",
+           "item", "marketGroup", "metaGroup", "unit", "alphaClones", "implantSet"]

eos/db/gamedata/alphaClones.py

@@ -0,0 +1,50 @@
+# ===============================================================================
+# Copyright (C) 2010 Diego Duclos
+#
+# This file is part of eos.
+#
+# eos is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# eos is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with eos.  If not, see <http://www.gnu.org/licenses/>.
+# ===============================================================================
+
+from sqlalchemy import Column, String, Integer, Table, ForeignKey
+from sqlalchemy.orm import relation, mapper, synonym
+
+from eos.db import gamedata_meta
+from eos.gamedata import AlphaClone, AlphaCloneSkill
+
+alphaclones_table = Table(
+        "alphaClones",
+        gamedata_meta,
+        Column("alphaCloneID", Integer, primary_key=True),
+        Column("alphaCloneName", String),
+)
+
+alphacloneskskills_table = Table(
+        "alphaCloneSkills",
+        gamedata_meta,
+        Column("alphaCloneID", Integer, ForeignKey("alphaClones.alphaCloneID"), primary_key=True),
+        Column("typeID", Integer, primary_key=True),
+        Column("level", Integer),
+)
+
+mapper(AlphaClone, alphaclones_table,
+       properties={
+           "ID"    : synonym("alphaCloneID"),
+           "skills": relation(
+                   AlphaCloneSkill,
+                   cascade="all,delete-orphan",
+                   backref="clone")
+       })
+
+mapper(AlphaCloneSkill, alphacloneskskills_table)

eos/db/gamedata/attribute.py

@@ -22,8 +22,8 @@ from sqlalchemy.ext.associationproxy import association_proxy
 from sqlalchemy.orm import relation, mapper, synonym, deferred
 
 from eos.db import gamedata_meta
-from eos.types import Attribute, Icon, AttributeInfo, Unit
-
+from eos.gamedata import Attribute, AttributeInfo, Unit
+import eos.config
 typeattributes_table = Table("dgmtypeattribs", gamedata_meta,
                              Column("value", Float),
                              Column("typeID", Integer, ForeignKey("invtypes.typeID"), primary_key=True, index=True),
@@ -36,26 +36,29 @@ attributes_table = Table("dgmattribs", gamedata_meta,
                          Column("maxAttributeID", Integer, ForeignKey("dgmattribs.attributeID")),
                          Column("description", Unicode),
                          Column("published", Boolean),
-                         Column("displayName", String),
+                         *[Column("displayName{}".format(lang), String) for lang in eos.config.translation_mapping.values()],
                          Column("highIsGood", Boolean),
-                         Column("iconID", Integer, ForeignKey("icons.iconID")),
+                         Column("iconID", Integer),
+                         Column("attributeCategory", Integer),
+                         # Column("tooltipDescription", Integer), # deprecated...?
                          Column("unitID", Integer, ForeignKey("dgmunits.unitID")))
 
 mapper(Attribute, typeattributes_table,
        properties={"info": relation(AttributeInfo, lazy=False)})
 
 mapper(AttributeInfo, attributes_table,
-       properties={"icon": relation(Icon),
-                   "unit": relation(Unit),
-                   "ID": synonym("attributeID"),
-                   "name": synonym("attributeName"),
-                   "description": deferred(attributes_table.c.description)})
+       properties={
+           "unit"       : relation(Unit),
+           "ID"         : synonym("attributeID"),
+           "name"       : synonym("attributeName"),
+           "description": deferred(attributes_table.c.description),
+       })
 
 Attribute.ID = association_proxy("info", "attributeID")
 Attribute.name = association_proxy("info", "attributeName")
 Attribute.description = association_proxy("info", "description")
 Attribute.published = association_proxy("info", "published")
-Attribute.displayName = association_proxy("info", "displayName")
+Attribute.displayName = association_proxy("info", "displayName{}".format(eos.config.lang))
 Attribute.highIsGood = association_proxy("info", "highIsGood")
 Attribute.iconID = association_proxy("info", "iconID")
 Attribute.icon = association_proxy("info", "icon")

eos/db/gamedata/category.py

@@ -17,21 +17,23 @@
 # along with eos.  If not, see <http://www.gnu.org/licenses/>.
 # ===============================================================================
 
-from sqlalchemy import Column, String, Integer, ForeignKey, Boolean, Table
-from sqlalchemy.orm import relation, mapper, synonym, deferred
+from sqlalchemy import Boolean, Column, Integer, String, Table
+from sqlalchemy.orm import deferred, mapper, synonym
 
 from eos.db import gamedata_meta
-from eos.types import Category, Icon
+from eos.gamedata import Category
+import eos.config
 
 categories_table = Table("invcategories", gamedata_meta,
                          Column("categoryID", Integer, primary_key=True),
-                         Column("categoryName", String),
-                         Column("description", String),
+                         *[Column("name{}".format(lang), String) for lang in eos.config.translation_mapping.values()],
+                         # Column("description", String), # deprecated
                          Column("published", Boolean),
-                         Column("iconID", Integer, ForeignKey("icons.iconID")))
+                         Column("iconID", Integer))
 
 mapper(Category, categories_table,
-       properties={"icon": relation(Icon),
-                   "ID": synonym("categoryID"),
-                   "name": synonym("categoryName"),
-                   "description": deferred(categories_table.c.description)})
+       properties={
+           "ID"         : synonym("categoryID"),
+           "displayName": synonym("name{}".format(eos.config.lang)),
+           # "description": deferred(categories_table.c.description) # deprecated
+       })

eos/db/gamedata/dynamicAttributes.py

@@ -0,0 +1,65 @@
+# ===============================================================================
+# Copyright (C) 2010 Diego Duclos
+#
+# This file is part of eos.
+#
+# eos is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# eos is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with eos.  If not, see <http://www.gnu.org/licenses/>.
+# ===============================================================================
+
+from sqlalchemy import Column, Float, Integer, Table, ForeignKey
+from sqlalchemy.orm import mapper, relation, synonym
+from sqlalchemy.ext.associationproxy import association_proxy
+
+from eos.db import gamedata_meta
+from eos.gamedata import DynamicItem, DynamicItemAttribute, DynamicItemItem, Item
+
+from eos.gamedata import AttributeInfo
+
+dynamic_table = Table("mutaplasmids", gamedata_meta,
+                          Column("typeID", ForeignKey("invtypes.typeID"), primary_key=True, index=True),
+                          Column("resultingTypeID", ForeignKey("invtypes.typeID"), primary_key=True))
+
+dynamicAttributes_table = Table("mutaplasmidAttributes", gamedata_meta,
+                      Column("typeID", Integer, ForeignKey("mutaplasmids.typeID"), primary_key=True),
+                      Column("attributeID", ForeignKey("dgmattribs.attributeID"), primary_key=True),
+                      Column("min", Float),
+                      Column("max", Float))
+
+dynamicApplicable_table = Table("mutaplasmidItems", gamedata_meta,
+                      Column("typeID", ForeignKey("mutaplasmids.typeID"), primary_key=True),
+                      Column("applicableTypeID", ForeignKey("invtypes.typeID"), primary_key=True),)
+
+mapper(DynamicItem, dynamic_table, properties={
+           "attributes": relation(DynamicItemAttribute),
+           "item": relation(Item, foreign_keys=[dynamic_table.c.typeID]),
+           "resultingItem": relation(Item, foreign_keys=[dynamic_table.c.resultingTypeID]),
+           "ID": synonym("typeID"),
+})
+
+mapper(DynamicItemAttribute, dynamicAttributes_table,
+       properties={"info": relation(AttributeInfo, lazy=False)})
+
+mapper(DynamicItemItem, dynamicApplicable_table, properties={
+           "mutaplasmid": relation(DynamicItem, viewonly=True),
+       })
+
+DynamicItemAttribute.ID = association_proxy("info", "attributeID")
+DynamicItemAttribute.name = association_proxy("info", "attributeName")
+DynamicItemAttribute.description = association_proxy("info", "description")
+DynamicItemAttribute.published = association_proxy("info", "published")
+DynamicItemAttribute.displayName = association_proxy("info", "displayName")
+DynamicItemAttribute.highIsGood = association_proxy("info", "highIsGood")
+DynamicItemAttribute.iconID = association_proxy("info", "iconID")
+DynamicItemAttribute.icon = association_proxy("info", "icon")
+DynamicItemAttribute.unit = association_proxy("info", "unit")

eos/db/gamedata/effect.py

@@ -18,11 +18,10 @@
 # ===============================================================================
 
 from sqlalchemy import Column, String, Integer, Boolean, Table, ForeignKey
-from sqlalchemy.ext.associationproxy import association_proxy
-from sqlalchemy.orm import mapper, synonym, relation, deferred
+from sqlalchemy.orm import mapper, synonym, deferred
 
 from eos.db import gamedata_meta
-from eos.types import Effect, EffectInfo
+from eos.gamedata import Effect, ItemEffect
 
 typeeffects_table = Table("dgmtypeeffects", gamedata_meta,
                           Column("typeID", Integer, ForeignKey("invtypes.typeID"), primary_key=True, index=True),
@@ -34,17 +33,14 @@ effects_table = Table("dgmeffects", gamedata_meta,
                       Column("description", String),
                       Column("published", Boolean),
                       Column("isAssistance", Boolean),
-                      Column("isOffensive", Boolean))
+                      Column("isOffensive", Boolean),
+                      Column("resistanceID", Integer))
 
-mapper(EffectInfo, effects_table,
-       properties={"ID": synonym("effectID"),
-                   "name": synonym("effectName"),
-                   "description": deferred(effects_table.c.description)})
+mapper(Effect, effects_table,
+       properties={
+           "ID"         : synonym("effectID"),
+           "name"       : synonym("effectName"),
+           "description": deferred(effects_table.c.description)
+       })
 
-mapper(Effect, typeeffects_table,
-       properties={"ID": synonym("effectID"),
-                   "info": relation(EffectInfo, lazy=False)})
-
-Effect.name = association_proxy("info", "name")
-Effect.description = association_proxy("info", "description")
-Effect.published = association_proxy("info", "published")
+mapper(ItemEffect, typeeffects_table)

eos/db/gamedata/group.py

@@ -18,22 +18,24 @@
 # ===============================================================================
 
 from sqlalchemy import Column, String, Integer, Boolean, ForeignKey, Table
-from sqlalchemy.orm import relation, mapper, synonym, deferred
+from sqlalchemy.orm import relation, mapper, synonym, deferred, backref
 
 from eos.db import gamedata_meta
-from eos.types import Group, Icon, Category
+from eos.gamedata import Category, Group
+import eos.config
 
 groups_table = Table("invgroups", gamedata_meta,
                      Column("groupID", Integer, primary_key=True),
-                     Column("groupName", String),
-                     Column("description", String),
+                     *[Column("name{}".format(lang), String) for lang in eos.config.translation_mapping.values()],
+                     # Column("description", String), # deprecated
                      Column("published", Boolean),
                      Column("categoryID", Integer, ForeignKey("invcategories.categoryID")),
-                     Column("iconID", Integer, ForeignKey("icons.iconID")))
+                     Column("iconID", Integer))
 
 mapper(Group, groups_table,
-       properties={"category": relation(Category, backref="groups"),
-                   "icon": relation(Icon),
-                   "ID": synonym("groupID"),
-                   "name": synonym("groupName"),
-                   "description": deferred(groups_table.c.description)})
+       properties={
+           "category"   : relation(Category, backref=backref("groups", cascade="all,delete")),
+           "ID"         : synonym("groupID"),
+           "displayName"       : synonym("name{}".format(eos.config.lang)),
+           # "description": deferred(groups_table.c.description) # deprecated
+       })

eos/db/gamedata/icon.py

@@ -1,33 +0,0 @@
-# ===============================================================================
-# Copyright (C) 2010 Diego Duclos
-#
-# This file is part of eos.
-#
-# eos is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation, either version 2 of the License, or
-# (at your option) any later version.
-#
-# eos is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with eos.  If not, see <http://www.gnu.org/licenses/>.
-# ===============================================================================
-
-from sqlalchemy import Column, String, Integer, Table
-from sqlalchemy.orm import mapper, synonym, deferred
-
-from eos.db import gamedata_meta
-from eos.types import Icon
-
-icons_table = Table("icons", gamedata_meta,
-                    Column("iconID", Integer, primary_key=True),
-                    Column("description", String),
-                    Column("iconFile", String))
-
-mapper(Icon, icons_table,
-       properties={"ID": synonym("iconID"),
-                   "description": deferred(icons_table.c.description)})

eos/db/gamedata/implantSet.py

@@ -0,0 +1,33 @@
+# ===============================================================================
+# Copyright (C) 2010 Diego Duclos
+#
+# This file is part of eos.
+#
+# eos is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# eos is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with eos.  If not, see <http://www.gnu.org/licenses/>.
+# ===============================================================================
+
+from sqlalchemy import Column, String, Integer, Table
+from sqlalchemy.orm import mapper, synonym
+
+from eos.db import gamedata_meta
+from eos.gamedata import ImplantSet
+
+implant_set_table = Table("implantsets", gamedata_meta,
+                          Column("setID", Integer, primary_key=True),
+                          Column("setName", String),
+                          Column("gradeName", String),
+                          Column("implants", String))
+
+mapper(ImplantSet, implant_set_table,
+       properties={"ID": synonym("setID")})

eos/db/gamedata/item.py

@@ -17,45 +17,67 @@
 # along with eos.  If not, see <http://www.gnu.org/licenses/>.
 # ===============================================================================
 
-from sqlalchemy import Column, String, Integer, Boolean, ForeignKey, Table, Float
+from sqlalchemy import Boolean, Column, ForeignKey, Integer, String, Table
 from sqlalchemy.ext.associationproxy import association_proxy
-from sqlalchemy.orm import relation, mapper, synonym, deferred
+from sqlalchemy.orm import backref, deferred, mapper, relation, synonym
 from sqlalchemy.orm.collections import attribute_mapped_collection
 
 from eos.db import gamedata_meta
-from eos.types import Icon, Attribute, Item, Effect, MetaType, Group, Traits
+from eos.db.gamedata.dynamicAttributes import dynamicApplicable_table
+from eos.db.gamedata.effect import typeeffects_table
+from eos.gamedata import Attribute, DynamicItem, Effect, Group, Item, Traits, MetaGroup
+
+import eos.config
 
 items_table = Table("invtypes", gamedata_meta,
                     Column("typeID", Integer, primary_key=True),
-                    Column("typeName", String, index=True),
-                    Column("description", String),
+                    *[Column("typeName{}".format(lang), String, index=True) for lang in eos.config.translation_mapping.values()],
+                    *[Column("typeDescription{}".format(lang), String) for lang in eos.config.translation_mapping.values()],
                     Column("raceID", Integer),
                     Column("factionID", Integer),
-                    Column("volume", Float),
-                    Column("mass", Float),
-                    Column("capacity", Float),
                     Column("published", Boolean),
                     Column("marketGroupID", Integer, ForeignKey("invmarketgroups.marketGroupID")),
-                    Column("iconID", Integer, ForeignKey("icons.iconID")),
-                    Column("groupID", Integer, ForeignKey("invgroups.groupID"), index=True))
+                    Column("iconID", Integer),
+                    Column("graphicID", Integer),
+                    Column("groupID", Integer, ForeignKey("invgroups.groupID"), index=True),
+                    Column("metaLevel", Integer),
+                    Column("metaGroupID", Integer, ForeignKey("invmetagroups.metaGroupID"), index=True),
+                    Column("variationParentTypeID", Integer, ForeignKey("invtypes.typeID"), index=True),
+                    Column("replacements", String),
+                    Column("reqskills", String),
+                    Column("requiredfor", String),
+                    )
 
-from .metaGroup import metatypes_table
-from .traits import traits_table
+from .traits import traits_table  # noqa
 
-mapper(Item, items_table,
-       properties={"group": relation(Group, backref="items"),
-                   "icon": relation(Icon),
-                   "_Item__attributes": relation(Attribute, collection_class=attribute_mapped_collection('name')),
-                   "effects": relation(Effect, collection_class=attribute_mapped_collection('name')),
-                   "metaGroup": relation(MetaType,
-                                         primaryjoin=metatypes_table.c.typeID == items_table.c.typeID,
-                                         uselist=False),
-                   "ID": synonym("typeID"),
-                   "name": synonym("typeName"),
-                   "description": deferred(items_table.c.description),
-                   "traits": relation(Traits,
-                                      primaryjoin=traits_table.c.typeID == items_table.c.typeID,
-                                      uselist=False)
-                   })
+props = {
+           "group": relation(Group, backref=backref("items", cascade="all,delete")),
+           "_Item__attributes": relation(Attribute, cascade='all, delete, delete-orphan', collection_class=attribute_mapped_collection('name')),
+           "effects": relation(Effect, secondary=typeeffects_table, collection_class=attribute_mapped_collection('name')),
+           "metaGroup": relation(MetaGroup, backref=backref("items", cascade="all,delete")),
+           "varParent": relation(Item, backref=backref("varChildren", cascade="all,delete"), remote_side=items_table.c.typeID),
+           "ID": synonym("typeID"),
+           "name": synonym("typeName{}".format(eos.config.lang)),
+           "description" : synonym("_description{}".format(eos.config.lang)),
+           "traits": relation(
+               Traits,
+               primaryjoin=traits_table.c.typeID == items_table.c.typeID,
+               uselist=False
+           ),
+           "mutaplasmids": relation(
+               DynamicItem,
+               primaryjoin=dynamicApplicable_table.c.applicableTypeID == items_table.c.typeID,
+               secondaryjoin=dynamicApplicable_table.c.typeID == DynamicItem.typeID,
+               secondary=dynamicApplicable_table,
+               backref="applicableItems",
+               viewonly=True
+           )
+}
+
+# Create deferred columns shadowing all the description fields. The literal `description` property will dynamically
+# be assigned as synonym to one of these
+props.update({'_description' + v: deferred(items_table.c['typeDescription' + v]) for (k, v) in eos.config.translation_mapping.items()})
+
+mapper(Item, items_table, properties=props)
 
 Item.category = association_proxy("group", "category")

eos/db/gamedata/marketGroup.py

@@ -21,22 +21,35 @@ from sqlalchemy import Column, String, Integer, Boolean, ForeignKey, Table
 from sqlalchemy.orm import relation, mapper, synonym, deferred
 
 from eos.db import gamedata_meta
-from eos.types import Item, MarketGroup, Icon
+from eos.gamedata import Item, MarketGroup
+import eos.config
 
 marketgroups_table = Table("invmarketgroups", gamedata_meta,
                            Column("marketGroupID", Integer, primary_key=True),
-                           Column("marketGroupName", String),
-                           Column("description", String),
+                           *[Column("marketGroupName{}".format(lang), String) for lang in eos.config.translation_mapping.values()],
+                           *[Column("marketGroupDescription{}".format(lang), String) for lang in eos.config.translation_mapping.values()],
                            Column("hasTypes", Boolean),
                            Column("parentGroupID", Integer,
-                                  ForeignKey("invmarketgroups.marketGroupID", initially="DEFERRED", deferrable=True)),
-                           Column("iconID", Integer, ForeignKey("icons.iconID")))
+                                ForeignKey("invmarketgroups.marketGroupID", initially="DEFERRED", deferrable=True)),
+                           Column("iconID", Integer))
+
+props = {
+    "items": relation(Item, backref="marketGroup"),
+    "parent": relation(MarketGroup, backref="children", remote_side=[marketgroups_table.c.marketGroupID]),
+    "ID": synonym("marketGroupID"),
+    "name": synonym("marketGroupName{}".format(eos.config.lang)),
+    "description": synonym("_description{}".format(eos.config.lang)),
+}
+
+# Create deferred columns shadowing all the description fields. The literal `description` property will dynamically
+# be assigned as synonym to one of these
+# this is mostly here to allow the db_update to be language-agnostic
+# todo: determine if we ever use market group descriptions... can we just get with of these?
+props.update({'_description' + v: deferred(marketgroups_table.c['marketGroupDescription' + v]) for (k, v) in eos.config.translation_mapping.items()})
+
+mapper(
+    MarketGroup,
+    marketgroups_table,
+    properties=props
+)
 
-mapper(MarketGroup, marketgroups_table,
-       properties={"items": relation(Item, backref="marketGroup"),
-                   "parent": relation(MarketGroup, backref="children",
-                                      remote_side=[marketgroups_table.c.marketGroupID]),
-                   "icon": relation(Icon),
-                   "ID": synonym("marketGroupID"),
-                   "name": synonym("marketGroupName"),
-                   "description": deferred(marketgroups_table.c.description)})

eos/db/gamedata/metaData.py

@@ -1,30 +1,30 @@
-# ===============================================================================
-# Copyright (C) 2010 Diego Duclos
-#
-# This file is part of eos.
-#
-# eos is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Lesser General Public License as published by
-# the Free Software Foundation, either version 2 of the License, or
-# (at your option) any later version.
-#
-# eos is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public License
-# along with eos.  If not, see <http://www.gnu.org/licenses/>.
-# ===============================================================================
-
-from sqlalchemy import Column, Table, String
-from sqlalchemy.orm import mapper
-
-from eos.db import gamedata_meta
-from eos.types import MetaData
-
-metadata_table = Table("metadata", gamedata_meta,
-                       Column("field_name", String, primary_key=True),
-                       Column("field_value", String))
-
-mapper(MetaData, metadata_table)
+# ===============================================================================
+# Copyright (C) 2010 Diego Duclos
+#
+# This file is part of eos.
+#
+# eos is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License as published by
+# the Free Software Foundation, either version 2 of the License, or
+# (at your option) any later version.
+#
+# eos is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with eos.  If not, see <http://www.gnu.org/licenses/>.
+# ===============================================================================
+
+from sqlalchemy import Column, Table, String
+from sqlalchemy.orm import mapper
+
+from eos.db import gamedata_meta
+from eos.gamedata import MetaData
+
+metadata_table = Table("metadata", gamedata_meta,
+                       Column("field_name", String, primary_key=True),
+                       Column("field_value", String))
+
+mapper(MetaData, metadata_table)

eos/db/gamedata/metaGroup.py

@@ -17,31 +17,25 @@
 # along with eos.  If not, see <http://www.gnu.org/licenses/>.
 # ===============================================================================
 
-from sqlalchemy import Table, Column, Integer, ForeignKey, String
-from sqlalchemy.ext.associationproxy import association_proxy
-from sqlalchemy.orm import relation, mapper, synonym
+from sqlalchemy import Table, Column, Integer, String
+from sqlalchemy.orm import mapper, synonym
 
 from eos.db import gamedata_meta
-from eos.db.gamedata.item import items_table
-from eos.types import MetaGroup, Item, MetaType
+from eos.gamedata import MetaGroup
+import eos.config
 
-metagroups_table = Table("invmetagroups", gamedata_meta,
-                         Column("metaGroupID", Integer, primary_key=True),
-                         Column("metaGroupName", String))
+metagroups_table = Table(
+    "invmetagroups",
+    gamedata_meta,
+    Column("metaGroupID", Integer, primary_key=True),
+    *[Column("metaGroupName{}".format(lang), String) for lang in eos.config.translation_mapping.values()],
+)
 
-metatypes_table = Table("invmetatypes", gamedata_meta,
-                        Column("typeID", Integer, ForeignKey("invtypes.typeID"), primary_key=True),
-                        Column("parentTypeID", Integer, ForeignKey("invtypes.typeID")),
-                        Column("metaGroupID", Integer, ForeignKey("invmetagroups.metaGroupID")))
-
-mapper(MetaGroup, metagroups_table,
-       properties={"ID": synonym("metaGroupID"),
-                   "name": synonym("metaGroupName")})
-
-mapper(MetaType, metatypes_table,
-       properties={"ID": synonym("metaGroupID"),
-                   "parent": relation(Item, primaryjoin=metatypes_table.c.parentTypeID == items_table.c.typeID),
-                   "items": relation(Item, primaryjoin=metatypes_table.c.typeID == items_table.c.typeID),
-                   "info": relation(MetaGroup, lazy=False)})
-
-MetaType.name = association_proxy("info", "name")
+mapper(
+    MetaGroup,
+    metagroups_table,
+    properties={
+        "ID"  : synonym("metaGroupID"),
+        "name": synonym("metaGroupName{}".format(eos.config.lang))
+    }
+)

eos/db/gamedata/queries.py

@@ -17,21 +17,22 @@
 # along with eos.  If not, see <http://www.gnu.org/licenses/>.
 # ===============================================================================
 
-from sqlalchemy.orm import join, exc
+from sqlalchemy.inspection import inspect
+from sqlalchemy.orm import aliased, exc, join
 from sqlalchemy.sql import and_, or_, select
 
 import eos.config
-from eos.db import gamedata_session
-from eos.db.gamedata.metaGroup import metatypes_table, items_table
+from eos.db import get_gamedata_session
+from eos.db.gamedata.item import items_table
+from eos.db.gamedata.group import groups_table
 from eos.db.util import processEager, processWhere
-from eos.types import Item, Category, Group, MarketGroup, AttributeInfo, MetaData, MetaGroup
+from eos.gamedata import AlphaClone, Attribute, AttributeInfo, Category, DynamicItem, Group, Item, MarketGroup, MetaData, MetaGroup, ImplantSet
 
+cache = {}
 configVal = getattr(eos.config, "gamedataCache", None)
 if configVal is True:
     def cachedQuery(amount, *keywords):
         def deco(function):
-            cache = {}
-
             def checkAndReturn(*args, **kwargs):
                 useCache = kwargs.pop("useCache", True)
                 cacheKey = []
@@ -63,7 +64,7 @@ else:
         return deco
 
 
-def sqlizeString(line):
+def sqlizeNormalString(line):
     # Escape backslashes first, as they will be as escape symbol in queries
     # Then escape percent and underscore signs
     # Finally, replace generic wildcards with sql-style wildcards
@@ -78,24 +79,112 @@ itemNameMap = {}
 def getItem(lookfor, eager=None):
     if isinstance(lookfor, int):
         if eager is None:
-            item = gamedata_session.query(Item).get(lookfor)
+            item = get_gamedata_session().query(Item).get(lookfor)
         else:
-            item = gamedata_session.query(Item).options(*processEager(eager)).filter(Item.ID == lookfor).first()
-    elif isinstance(lookfor, basestring):
+            item = get_gamedata_session().query(Item).options(*processEager(eager)).filter(Item.ID == lookfor).first()
+    elif isinstance(lookfor, str):
         if lookfor in itemNameMap:
             id = itemNameMap[lookfor]
             if eager is None:
-                item = gamedata_session.query(Item).get(id)
+                item = get_gamedata_session().query(Item).get(id)
             else:
-                item = gamedata_session.query(Item).options(*processEager(eager)).filter(Item.ID == id).first()
+                item = get_gamedata_session().query(Item).options(*processEager(eager)).filter(Item.ID == id).first()
         else:
             # Item names are unique, so we can use first() instead of one()
-            item = gamedata_session.query(Item).options(*processEager(eager)).filter(Item.name == lookfor).first()
-            itemNameMap[lookfor] = item.ID
+            item = get_gamedata_session().query(Item).options(*processEager(eager)).filter(Item.typeName == lookfor).first()
+            if item is not None:
+                itemNameMap[lookfor] = item.ID
     else:
         raise TypeError("Need integer or string as argument")
     return item
 
+@cachedQuery(1, "itemIDs")
+def getItems(itemIDs, eager=None):
+    if not isinstance(itemIDs, (tuple, list, set)) or not all(isinstance(t, int) for t in itemIDs):
+        raise TypeError("Need iterable of integers as argument")
+    if eager is None:
+        items = get_gamedata_session().query(Item).filter(Item.ID.in_(itemIDs)).all()
+    else:
+        items = get_gamedata_session().query(Item).options(*processEager(eager)).filter(Item.ID.in_(itemIDs)).all()
+    return items
+
+
+def getMutaplasmid(lookfor, eager=None):
+    if isinstance(lookfor, int):
+        item = get_gamedata_session().query(DynamicItem).filter(DynamicItem.ID == lookfor).first()
+    else:
+        raise TypeError("Need integer as argument")
+    return item
+
+
+def getItemWithBaseItemAttribute(lookfor, baseItemID, eager=None):
+    # A lot of this is described in more detail in #1597
+    item = get_gamedata_session().query(Item).get(lookfor)
+    base = getItem(baseItemID)
+
+    # we have to load all attributes for this object, otherwise we'll lose access to them when we expunge.
+    # todo: figure out a way to eagerly load all these via the query...
+    for x in [*inspect(Item).relationships.keys(), 'description']:
+        getattr(item, x)
+
+    # Copy over the attributes from the base, but ise the items attributes when there's an overlap
+    # WARNING: the attribute object still has the old typeID. I don't believe we access this typeID anywhere in the code,
+    # but should keep this in mind for now.
+    item._Item__attributes = {**base.attributes, **item.attributes}
+
+    # Expunge the item form the session. This is required to have different Abyssal / Base combinations loaded in memory.
+    # Without expunging it, once one Abyssal Web is created, SQLAlchmey will use it for all others. We don't want this,
+    # we want to generate a completely new object to work with
+    get_gamedata_session().expunge(item)
+    return item
+
+
+@cachedQuery(1, "lookfor")
+def getItems(lookfor, eager=None):
+    """
+    Gets a list of items. Does a bit of cache hackery to get working properly -- cache
+    is usually based on function calls with the parameters, needed to extract data directly.
+    Works well enough. Not currently used, but it's here for possible future inclusion
+    """
+
+    toGet = []
+    results = []
+
+    for id in lookfor:
+        if (id, None) in cache:
+            results.append(cache.get((id, None)))
+        else:
+            toGet.append(id)
+
+    if len(toGet) > 0:
+        # Get items that aren't currently cached, and store them in the cache
+        items = get_gamedata_session().query(Item).filter(Item.ID.in_(toGet)).all()
+        for item in items:
+            cache[(item.ID, None)] = item
+        results += items
+
+    # sort the results based on the original indexing
+    results.sort(key=lambda x: lookfor.index(x.ID))
+    return results
+
+
+@cachedQuery(1, "lookfor")
+def getAlphaClone(lookfor, eager=None):
+    if isinstance(lookfor, int):
+        if eager is None:
+            item = get_gamedata_session().query(AlphaClone).get(lookfor)
+        else:
+            item = get_gamedata_session().query(AlphaClone).options(*processEager(eager)).filter(AlphaClone.ID == lookfor).first()
+    else:
+        raise TypeError("Need integer as argument")
+    return item
+
+
+def getAlphaCloneList(eager=None):
+    eager = processEager(eager)
+    clones = get_gamedata_session().query(AlphaClone).options(*eager).all()
+    return clones
+
 
 groupNameMap = {}
 
@@ -104,20 +193,21 @@ groupNameMap = {}
 def getGroup(lookfor, eager=None):
     if isinstance(lookfor, int):
         if eager is None:
-            group = gamedata_session.query(Group).get(lookfor)
+            group = get_gamedata_session().query(Group).get(lookfor)
         else:
-            group = gamedata_session.query(Group).options(*processEager(eager)).filter(Group.ID == lookfor).first()
-    elif isinstance(lookfor, basestring):
+            group = get_gamedata_session().query(Group).options(*processEager(eager)).filter(Group.ID == lookfor).first()
+    elif isinstance(lookfor, str):
         if lookfor in groupNameMap:
             id = groupNameMap[lookfor]
             if eager is None:
-                group = gamedata_session.query(Group).get(id)
+                group = get_gamedata_session().query(Group).get(id)
             else:
-                group = gamedata_session.query(Group).options(*processEager(eager)).filter(Group.ID == id).first()
+                group = get_gamedata_session().query(Group).options(*processEager(eager)).filter(Group.ID == id).first()
         else:
             # Group names are unique, so we can use first() instead of one()
-            group = gamedata_session.query(Group).options(*processEager(eager)).filter(Group.name == lookfor).first()
-            groupNameMap[lookfor] = group.ID
+            group = get_gamedata_session().query(Group).options(*processEager(eager)).filter(Group.name == lookfor).first()
+            if group is not None:
+                groupNameMap[lookfor] = group.ID
     else:
         raise TypeError("Need integer or string as argument")
     return group
@@ -130,23 +220,24 @@ categoryNameMap = {}
 def getCategory(lookfor, eager=None):
     if isinstance(lookfor, int):
         if eager is None:
-            category = gamedata_session.query(Category).get(lookfor)
+            category = get_gamedata_session().query(Category).get(lookfor)
         else:
-            category = gamedata_session.query(Category).options(*processEager(eager)).filter(
-                Category.ID == lookfor).first()
-    elif isinstance(lookfor, basestring):
+            category = get_gamedata_session().query(Category).options(*processEager(eager)).filter(
+                    Category.ID == lookfor).first()
+    elif isinstance(lookfor, str):
         if lookfor in categoryNameMap:
             id = categoryNameMap[lookfor]
             if eager is None:
-                category = gamedata_session.query(Category).get(id)
+                category = get_gamedata_session().query(Category).get(id)
             else:
-                category = gamedata_session.query(Category).options(*processEager(eager)).filter(
-                    Category.ID == id).first()
+                category = get_gamedata_session().query(Category).options(*processEager(eager)).filter(
+                        Category.ID == id).first()
         else:
             # Category names are unique, so we can use first() instead of one()
-            category = gamedata_session.query(Category).options(*processEager(eager)).filter(
-                Category.name == lookfor).first()
-            categoryNameMap[lookfor] = category.ID
+            category = get_gamedata_session().query(Category).options(*processEager(eager)).filter(
+                    Category.name == lookfor).first()
+            if category is not None:
+                categoryNameMap[lookfor] = category.ID
     else:
         raise TypeError("Need integer or string as argument")
     return category
@@ -159,58 +250,72 @@ metaGroupNameMap = {}
 def getMetaGroup(lookfor, eager=None):
     if isinstance(lookfor, int):
         if eager is None:
-            metaGroup = gamedata_session.query(MetaGroup).get(lookfor)
+            metaGroup = get_gamedata_session().query(MetaGroup).get(lookfor)
         else:
-            metaGroup = gamedata_session.query(MetaGroup).options(*processEager(eager)).filter(
-                MetaGroup.ID == lookfor).first()
-    elif isinstance(lookfor, basestring):
+            metaGroup = get_gamedata_session().query(MetaGroup).options(*processEager(eager)).filter(
+                    MetaGroup.ID == lookfor).first()
+    elif isinstance(lookfor, str):
         if lookfor in metaGroupNameMap:
             id = metaGroupNameMap[lookfor]
             if eager is None:
-                metaGroup = gamedata_session.query(MetaGroup).get(id)
+                metaGroup = get_gamedata_session().query(MetaGroup).get(id)
             else:
-                metaGroup = gamedata_session.query(MetaGroup).options(*processEager(eager)).filter(
-                    MetaGroup.ID == id).first()
+                metaGroup = get_gamedata_session().query(MetaGroup).options(*processEager(eager)).filter(
+                        MetaGroup.ID == id).first()
         else:
             # MetaGroup names are unique, so we can use first() instead of one()
-            metaGroup = gamedata_session.query(MetaGroup).options(*processEager(eager)).filter(
-                MetaGroup.name == lookfor).first()
-            metaGroupNameMap[lookfor] = metaGroup.ID
+            metaGroup = get_gamedata_session().query(MetaGroup).options(*processEager(eager)).filter(
+                    MetaGroup.metaGroupName == lookfor).first()
+            if metaGroup is not None:
+                metaGroupNameMap[lookfor] = metaGroup.ID
     else:
         raise TypeError("Need integer or string as argument")
     return metaGroup
 
 
+def getMetaGroups():
+    return get_gamedata_session().query(MetaGroup).all()
+
+
 @cachedQuery(1, "lookfor")
 def getMarketGroup(lookfor, eager=None):
     if isinstance(lookfor, int):
         if eager is None:
-            marketGroup = gamedata_session.query(MarketGroup).get(lookfor)
+            marketGroup = get_gamedata_session().query(MarketGroup).get(lookfor)
         else:
-            marketGroup = gamedata_session.query(MarketGroup).options(*processEager(eager)).filter(
-                MarketGroup.ID == lookfor).first()
+            marketGroup = get_gamedata_session().query(MarketGroup).options(*processEager(eager)).filter(
+                    MarketGroup.ID == lookfor).first()
     else:
         raise TypeError("Need integer as argument")
     return marketGroup
 
 
+def getMarketTreeNodeIds(rootNodeIds):
+    allIds = set()
+    addedIds = set(rootNodeIds)
+    while addedIds:
+        allIds.update(addedIds)
+        addedIds = {mg.ID for mg in get_gamedata_session().query(MarketGroup).filter(MarketGroup.parentGroupID.in_(addedIds))}
+    return allIds
+
+
 @cachedQuery(2, "where", "filter")
 def getItemsByCategory(filter, where=None, eager=None):
     if isinstance(filter, int):
         filter = Category.ID == filter
-    elif isinstance(filter, basestring):
+    elif isinstance(filter, str):
         filter = Category.name == filter
     else:
         raise TypeError("Need integer or string as argument")
 
     filter = processWhere(filter, where)
-    return gamedata_session.query(Item).options(*processEager(eager)).join(Item.group, Group.category).filter(
-        filter).all()
+    return get_gamedata_session().query(Item).options(*processEager(eager)).join(Item.group, Group.category).filter(
+            filter).all()
 
 
 @cachedQuery(3, "where", "nameLike", "join")
 def searchItems(nameLike, where=None, join=None, eager=None):
-    if not isinstance(nameLike, basestring):
+    if not isinstance(nameLike, str):
         raise TypeError("Need string as argument")
 
     if join is None:
@@ -219,9 +324,9 @@ def searchItems(nameLike, where=None, join=None, eager=None):
     if not hasattr(join, "__iter__"):
         join = (join,)
 
-    items = gamedata_session.query(Item).options(*processEager(eager)).join(*join)
+    items = get_gamedata_session().query(Item).options(*processEager(eager)).join(*join)
     for token in nameLike.split(' '):
-        token_safe = u"%{0}%".format(sqlizeString(token))
+        token_safe = "%{0}%".format(sqlizeNormalString(token))
         if where is not None:
             items = items.filter(and_(Item.name.like(token_safe, escape="\\"), where))
         else:
@@ -230,8 +335,45 @@ def searchItems(nameLike, where=None, join=None, eager=None):
     return items
 
 
+@cachedQuery(3, "tokens", "where", "join")
+def searchItemsRegex(tokens, where=None, join=None, eager=None):
+    if not isinstance(tokens, (tuple, list)) or not all(isinstance(t, str) for t in tokens):
+        raise TypeError("Need tuple or list of strings as argument")
+
+    if join is None:
+        join = tuple()
+
+    if not hasattr(join, "__iter__"):
+        join = (join,)
+
+    items = get_gamedata_session().query(Item).options(*processEager(eager)).join(*join)
+    for token in tokens:
+        if where is not None:
+            items = items.filter(and_(Item.name.op('regexp')(token), where))
+        else:
+            items = items.filter(Item.name.op('regexp')(token))
+    items = items.limit(100).all()
+    return items
+
+
+@cachedQuery(3, "where", "nameLike", "join")
+def searchSkills(nameLike, where=None, eager=None):
+    if not isinstance(nameLike, str):
+        raise TypeError("Need string as argument")
+
+    items = get_gamedata_session().query(Item).options(*processEager(eager)).join(Item.group, Group.category)
+    for token in nameLike.split(' '):
+        token_safe = "%{0}%".format(sqlizeNormalString(token))
+        if where is not None:
+            items = items.filter(and_(Item.name.like(token_safe, escape="\\"), Category.ID == 16, where))
+        else:
+            items = items.filter(and_(Item.name.like(token_safe, escape="\\"), Category.ID == 16))
+    items = items.limit(100).all()
+    return items
+
+
 @cachedQuery(2, "where", "itemids")
-def getVariations(itemids, where=None, eager=None):
+def getVariations(itemids, groupIDs=None, where=None, eager=None):
     for itemid in itemids:
         if not isinstance(itemid, int):
             raise TypeError("All passed item IDs must be integers")
@@ -239,24 +381,32 @@ def getVariations(itemids, where=None, eager=None):
     if len(itemids) == 0:
         return []
 
-    itemfilter = or_(*(metatypes_table.c.parentTypeID == itemid for itemid in itemids))
+    itemfilter = or_(*(items_table.c.variationParentTypeID == itemid for itemid in itemids))
     filter = processWhere(itemfilter, where)
-    joinon = items_table.c.typeID == metatypes_table.c.typeID
-    vars = gamedata_session.query(Item).options(*processEager(eager)).join((metatypes_table, joinon)).filter(
-        filter).all()
+    vars = get_gamedata_session().query(Item).options(*processEager(eager)).filter(filter).all()
+
+    if vars:
+        return vars
+    elif groupIDs:
+        itemfilter = or_(*(groups_table.c.groupID == groupID for groupID in groupIDs))
+        filter = processWhere(itemfilter, where)
+        joinon = items_table.c.groupID == groups_table.c.groupID
+        vars = get_gamedata_session().query(Item).options(*processEager(eager)).join((groups_table, joinon)).filter(
+                filter).all()
+
     return vars
 
 
 @cachedQuery(1, "attr")
 def getAttributeInfo(attr, eager=None):
-    if isinstance(attr, basestring):
+    if isinstance(attr, str):
         filter = AttributeInfo.name == attr
     elif isinstance(attr, int):
         filter = AttributeInfo.ID == attr
     else:
         raise TypeError("Need integer or string as argument")
     try:
-        result = gamedata_session.query(AttributeInfo).options(*processEager(eager)).filter(filter).one()
+        result = get_gamedata_session().query(AttributeInfo).options(*processEager(eager)).filter(filter).one()
     except exc.NoResultFound:
         result = None
     return result
@@ -264,8 +414,8 @@ def getAttributeInfo(attr, eager=None):
 
 @cachedQuery(1, "field")
 def getMetaData(field):
-    if isinstance(field, basestring):
-        data = gamedata_session.query(MetaData).get(field)
+    if isinstance(field, str):
+        data = get_gamedata_session().query(MetaData).get(field)
     else:
         raise TypeError("Need string as argument")
     return data
@@ -280,9 +430,34 @@ def directAttributeRequest(itemIDs, attrIDs):
         if not isinstance(itemID, int):
             raise TypeError("All itemIDs must be integer")
 
-    q = select((eos.types.Item.typeID, eos.types.Attribute.attributeID, eos.types.Attribute.value),
-               and_(eos.types.Attribute.attributeID.in_(attrIDs), eos.types.Item.typeID.in_(itemIDs)),
-               from_obj=[join(eos.types.Attribute, eos.types.Item)])
+    q = select((Item.typeID, Attribute.attributeID, Attribute.value),
+               and_(Attribute.attributeID.in_(attrIDs), Item.typeID.in_(itemIDs)),
+               from_obj=[join(Attribute, Item)])
 
-    result = gamedata_session.execute(q).fetchall()
+    result = get_gamedata_session().execute(q).fetchall()
     return result
+
+
+def getAbyssalTypes():
+    return set([r.resultingTypeID for r in get_gamedata_session().query(DynamicItem.resultingTypeID).distinct()])
+
+
+@cachedQuery(1, "itemID")
+def getDynamicItem(itemID, eager=None):
+    try:
+        if isinstance(itemID, int):
+            if eager is None:
+                result = get_gamedata_session().query(DynamicItem).filter(DynamicItem.ID == itemID).one()
+            else:
+                result = get_gamedata_session().query(DynamicItem).options(*processEager(eager)).filter(DynamicItem.ID == itemID).one()
+        else:
+            raise TypeError("Need integer as argument")
+    except exc.NoResultFound:
+        result = None
+    return result
+
+
+@cachedQuery(1, "lookfor")
+def getAllImplantSets():
+    implantSets = get_gamedata_session().query(ImplantSet).all()
+    return implantSets

eos/db/gamedata/traits.py

@@ -1,11 +1,21 @@
 from sqlalchemy import Column, Table, Integer, String, ForeignKey
-from sqlalchemy.orm import mapper
+from sqlalchemy.orm import mapper, synonym
 
 from eos.db import gamedata_meta
-from eos.types import Traits
+from eos.gamedata import Traits
+import eos.config
 
-traits_table = Table("invtraits", gamedata_meta,
-                     Column("typeID", Integer, ForeignKey("invtypes.typeID"), primary_key=True),
-                     Column("traitText", String))
+traits_table = Table(
+    "invtraits",
+    gamedata_meta,
+    Column("typeID", Integer, ForeignKey("invtypes.typeID"), primary_key=True),
+    *[Column("traitText{}".format(lang), String) for lang in eos.config.translation_mapping.values()],
+)
 
-mapper(Traits, traits_table)
+mapper(
+    Traits,
+    traits_table,
+    properties={
+        "display": synonym("traitText{}".format(eos.config.lang)),
+    }
+)

eos/db/gamedata/unit.py

@@ -21,13 +21,17 @@ from sqlalchemy import Column, Table, Integer, String
 from sqlalchemy.orm import mapper, synonym
 
 from eos.db import gamedata_meta
-from eos.types import Unit
+from eos.gamedata import Unit
+import eos.config
 
 groups_table = Table("dgmunits", gamedata_meta,
                      Column("unitID", Integer, primary_key=True),
                      Column("unitName", String),
-                     Column("displayName", String))
+                     *[Column("displayName{}".format(lang), String) for lang in eos.config.translation_mapping.values()],
+                     )
 
 mapper(Unit, groups_table,
-       properties={"ID": synonym("unitID"),
-                   "name": synonym("unitName")})
+       properties={
+           "ID"  : synonym("unitID"),
+           "name": synonym("unitName")
+        })

eos/db/migration.py

@@ -1,11 +1,11 @@
-import logging
+from logbook import Logger
 import shutil
 import time
 
 import config
-import migrations
+from . import migrations
 
-logger = logging.getLogger(__name__)
+pyfalog = Logger(__name__)
 
 
 def getVersion(db):
@@ -34,10 +34,10 @@ def update(saveddata_engine):
 
         shutil.copyfile(config.saveDB, toFile)
 
-        for version in xrange(dbVersion, appVersion):
+        for version in range(dbVersion, appVersion):
             func = migrations.updates[version + 1]
             if func:
-                logger.info("Applying database update: %d", version + 1)
+                pyfalog.info("Applying database update: {0}", version + 1)
                 func(saveddata_engine)
 
         # when all is said and done, set version to current

eos/db/migrations/__init__.py

@@ -8,23 +8,25 @@ many upgrade files as there are database versions (version 5 would include
 upgrade files 1-5)
 """
 
-import pkgutil
 import re
 
+from eos.utils.pyinst_support import iterNamespace
+
 updates = {}
 appVersion = 0
 
 prefix = __name__ + "."
-for importer, modname, ispkg in pkgutil.iter_modules(__path__, prefix):
+
+for modName in iterNamespace(__name__, __path__):
     # loop through python files, extracting update number and function, and
     # adding it to a list
-    modname_tail = modname.rsplit('.', 1)[-1]
-    module = __import__(modname, fromlist=True)
-    m = re.match("^upgrade(?P<index>\d+)$", modname_tail)
+    modname_tail = modName.rsplit('.', 1)[-1]
+    m = re.match(r"^upgrade(?P<index>\d+)$", modname_tail)
     if not m:
         continue
     index = int(m.group("index"))
     appVersion = max(appVersion, index)
+    module = __import__(modName, fromlist=True)
     upgrade = getattr(module, "upgrade", False)
     if upgrade:
         updates[index] = upgrade

eos/db/migrations/upgrade1.py

@@ -7,57 +7,57 @@ Migration 1
     loaded as they no longer exist in the database. We therefore replace these
     modules with their new replacements
 
-    Based on http://community.eveonline.com/news/patch-notes/patch-notes-for-oceanus/
+    Based on https://www.eveonline.com/news/view/patch-notes-for-oceanus
     and output of itemDiff.py
 """
 
 import sqlalchemy
 
 CONVERSIONS = {
-    6135: [  # Scoped Cargo Scanner
+    6135 : [  # Scoped Cargo Scanner
         6133,  # Interior Type-E Cargo Identifier
     ],
-    6527: [  # Compact Ship Scanner
+    6527 : [  # Compact Ship Scanner
         6525,  # Ta3 Perfunctory Vessel Probe
         6529,  # Speculative Ship Identifier I
         6531,  # Practical Type-E Ship Probe
     ],
-    6569: [  # Scoped Survey Scanner
+    6569 : [  # Scoped Survey Scanner
         6567,  # ML-3 Amphilotite Mining Probe
         6571,  # Rock-Scanning Sensor Array I
         6573,  # 'Dactyl' Type-E Asteroid Analyzer
     ],
-    509: [  # 'Basic' Capacitor Flux Coil
+    509  : [  # 'Basic' Capacitor Flux Coil
         8163,  # Partial Power Plant Manager: Capacitor Flux
         8165,  # Alpha Reactor Control: Capacitor Flux
         8167,  # Type-E Power Core Modification: Capacitor Flux
         8169,  # Marked Generator Refitting: Capacitor Flux
     ],
-    8135: [  # Restrained Capacitor Flux Coil
+    8135 : [  # Restrained Capacitor Flux Coil
         8131,  # Local Power Plant Manager: Capacitor Flux I
     ],
-    8133: [  # Compact Capacitor Flux Coil
+    8133 : [  # Compact Capacitor Flux Coil
         8137,  # Mark I Generator Refitting: Capacitor Flux
     ],
-    3469: [  # Basic Co-Processor
+    3469 : [  # Basic Co-Processor
         8744,  # Nanoelectrical Co-Processor
         8743,  # Nanomechanical CPU Enhancer
         8746,  # Quantum Co-Processor
         8745,  # Photonic CPU Enhancer
         15425,  # Naiyon's Modified Co-Processor (never existed but convert
-                # anyway as some fits may include it)
+        # anyway as some fits may include it)
     ],
-    8748: [  # Upgraded Co-Processor
+    8748 : [  # Upgraded Co-Processor
         8747,  # Nanomechanical CPU Enhancer I
         8750,  # Quantum Co-Processor I
         8749,  # Photonic CPU Enhancer I
     ],
-    1351: [  # Basic Reactor Control Unit
+    1351 : [  # Basic Reactor Control Unit
         8251,  # Partial Power Plant Manager: Reaction Control
         8253,  # Alpha Reactor Control: Reaction Control
         8257,  # Marked Generator Refitting: Reaction Control
     ],
-    8263: [  # Compact Reactor Control Unit
+    8263 : [  # Compact Reactor Control Unit
         8259,  # Local Power Plant Manager: Reaction Control I
         8265,  # Mark I Generator Refitting: Reaction Control
         8261,  # Beta Reactor Control: Reaction Control I
@@ -69,19 +69,20 @@ CONVERSIONS = {
     31936: [  # Navy Micro Auxiliary Power Core
         16543,  # Micro 'Vigor' Core Augmentation
     ],
-    8089: [  # Compact Light Missile Launcher
-       8093,  # Prototype 'Arbalest' Light Missile Launcher
+    8089 : [  # Compact Light Missile Launcher
+        8093,  # Prototype 'Arbalest' Light Missile Launcher
     ],
-    8091: [  # Ample Light Missile Launcher
+    8091 : [  # Ample Light Missile Launcher
         7993,  # Experimental TE-2100 Light Missile Launcher
     ],
     # Surface Cargo Scanner I was removed from game, however no mention of
     # replacement module in patch notes. Morphing it to meta 0 module to be safe
-    442: [  # Cargo Scanner I
+    442  : [  # Cargo Scanner I
         6129,  # Surface Cargo Scanner I
     ]
 }
 
+
 def upgrade(saveddata_engine):
     # Update fits schema to include target resists attribute
     try:
@@ -90,8 +91,9 @@ def upgrade(saveddata_engine):
         saveddata_engine.execute("ALTER TABLE fits ADD COLUMN targetResistsID INTEGER;")
 
     # Convert modules
-    for replacement_item, list in CONVERSIONS.iteritems():
+    for replacement_item, list in CONVERSIONS.items():
         for retired_item in list:
-            saveddata_engine.execute('UPDATE "modules" SET "itemID" = ? WHERE "itemID" = ?', (replacement_item, retired_item))
-            saveddata_engine.execute('UPDATE "cargo" SET "itemID" = ? WHERE "itemID" = ?', (replacement_item, retired_item))
-
+            saveddata_engine.execute('UPDATE "modules" SET "itemID" = ? WHERE "itemID" = ?',
+                                     (replacement_item, retired_item))
+            saveddata_engine.execute('UPDATE "cargo" SET "itemID" = ? WHERE "itemID" = ?',
+                                     (replacement_item, retired_item))

eos/db/migrations/upgrade10.py

@@ -6,6 +6,7 @@ Migration 10
 
 import sqlalchemy
 
+
 def upgrade(saveddata_engine):
     # Update projectedFits schema to include active attribute
     try:

eos/db/migrations/upgrade11.py

@@ -7,7 +7,6 @@ Migration 11
     modules with their new replacements
 """
 
-
 CONVERSIONS = {
     16467: (  # Medium Gremlin Compact Energy Neutralizer
         16471,  # Medium Unstable Power Fluctuator I
@@ -15,7 +14,7 @@ CONVERSIONS = {
     22947: (  # 'Beatnik' Small Remote Armor Repairer
         23414,  # 'Brotherhood' Small Remote Armor Repairer
     ),
-    8295: (  # Type-D Restrained Shield Flux Coil
+    8295 : (  # Type-D Restrained Shield Flux Coil
         8293,  # Beta Reactor Control: Shield Flux I
     ),
     16499: (  # Heavy Knave Scoped Energy Nosferatu
@@ -30,13 +29,13 @@ CONVERSIONS = {
     16447: (  # Medium Solace Scoped Remote Armor Repairer
         16445,  # Medium 'Arup' Remote Armor Repairer
     ),
-    508: (  # 'Basic' Shield Flux Coil
+    508  : (  # 'Basic' Shield Flux Coil
         8325,  # Alpha Reactor Shield Flux
         8329,  # Marked Generator Refitting: Shield Flux
         8323,  # Partial Power Plant Manager: Shield Flux
         8327,  # Type-E Power Core Modification: Shield Flux
     ),
-    1419: (  # 'Basic' Shield Power Relay
+    1419 : (  # 'Basic' Shield Power Relay
         8341,  # Alpha Reactor Shield Power Relay
         8345,  # Marked Generator Refitting: Shield Power Relay
         8339,  # Partial Power Plant Manager: Shield Power Relay
@@ -48,57 +47,57 @@ CONVERSIONS = {
     16505: (  # Medium Ghoul Compact Energy Nosferatu
         16511,  # Medium Diminishing Power System Drain I
     ),
-    8297: (  # Mark I Compact Shield Flux Coil
+    8297 : (  # Mark I Compact Shield Flux Coil
         8291,  # Local Power Plant Manager: Reaction Shield Flux I
     ),
     16455: (  # Large Solace Scoped Remote Armor Repairer
         16453,  # Large 'Arup' Remote Armor Repairer
     ),
-    6485: (  # M51 Benefactor Compact Shield Recharger
+    6485 : (  # M51 Benefactor Compact Shield Recharger
         6491,  # Passive Barrier Compensator I
         6489,  # 'Benefactor' Ward Reconstructor
         6487,  # Supplemental Screen Generator I
     ),
-    5137: (  # Small Knave Scoped Energy Nosferatu
+    5137 : (  # Small Knave Scoped Energy Nosferatu
         5135,  # E5 Prototype Energy Vampire
     ),
-    8579: (  # Medium Murky Compact Remote Shield Booster
+    8579 : (  # Medium Murky Compact Remote Shield Booster
         8581,  # Medium 'Atonement' Remote Shield Booster
     ),
-    8531: (  # Small Murky Compact Remote Shield Booster
+    8531 : (  # Small Murky Compact Remote Shield Booster
         8533,  # Small 'Atonement' Remote Shield Booster
     ),
     16497: (  # Heavy Ghoul Compact Energy Nosferatu
         16503,  # Heavy Diminishing Power System Drain I
     ),
-    4477: (  # Small Gremlin Compact Energy Neutralizer
+    4477 : (  # Small Gremlin Compact Energy Neutralizer
         4475,  # Small Unstable Power Fluctuator I
     ),
-    8337: (  # Mark I Compact Shield Power Relay
+    8337 : (  # Mark I Compact Shield Power Relay
         8331,  # Local Power Plant Manager: Reaction Shield Power Relay I
     ),
     23416: (  # 'Peace' Large Remote Armor Repairer
         22951,  # 'Pacifier' Large Remote Armor Repairer
     ),
-    5141: (  # Small Ghoul Compact Energy Nosferatu
+    5141 : (  # Small Ghoul Compact Energy Nosferatu
         5139,  # Small Diminishing Power System Drain I
     ),
-    4471: (  # Small Infectious Scoped Energy Neutralizer
+    4471 : (  # Small Infectious Scoped Energy Neutralizer
         4473,  # Small Rudimentary Energy Destabilizer I
     ),
     16469: (  # Medium Infectious Scoped Energy Neutralizer
         16465,  # Medium Rudimentary Energy Destabilizer I
     ),
-    8335: (  # Type-D Restrained Shield Power Relay
+    8335 : (  # Type-D Restrained Shield Power Relay
         8333,  # Beta Reactor Control: Shield Power Relay I
     ),
-    405: (  # 'Micro' Remote Shield Booster
+    405  : (  # 'Micro' Remote Shield Booster
         8631,  # Micro Asymmetric Remote Shield Booster
         8627,  # Micro Murky Remote Shield Booster
         8629,  # Micro 'Atonement' Remote Shield Booster
         8633,  # Micro S95a Remote Shield Booster
     ),
-    8635: (  # Large Murky Compact Remote Shield Booster
+    8635 : (  # Large Murky Compact Remote Shield Booster
         8637,  # Large 'Atonement' Remote Shield Booster
     ),
     16507: (  # Medium Knave Scoped Energy Nosferatu
@@ -106,11 +105,12 @@ CONVERSIONS = {
     ),
 }
 
+
 def upgrade(saveddata_engine):
-
     # Convert modules
-    for replacement_item, list in CONVERSIONS.iteritems():
+    for replacement_item, list in CONVERSIONS.items():
         for retired_item in list:
-            saveddata_engine.execute('UPDATE "modules" SET "itemID" = ? WHERE "itemID" = ?', (replacement_item, retired_item))
-            saveddata_engine.execute('UPDATE "cargo" SET "itemID" = ? WHERE "itemID" = ?', (replacement_item, retired_item))
-
+            saveddata_engine.execute('UPDATE "modules" SET "itemID" = ? WHERE "itemID" = ?',
+                                     (replacement_item, retired_item))
+            saveddata_engine.execute('UPDATE "cargo" SET "itemID" = ? WHERE "itemID" = ?',
+                                     (replacement_item, retired_item))

eos/db/migrations/upgrade12.py

@@ -7,33 +7,32 @@ Migration 12
     modules with their new replacements
 """
 
-
 CONVERSIONS = {
     16457: (  # Crosslink Compact Ballistic Control System
         16459,  # Muon Coil Bolt Array I
         16461,  # Multiphasic Bolt Array I
         16463,  # 'Pandemonium' Ballistic Enhancement
     ),
-    5281: (  # Coadjunct Scoped Remote Sensor Booster
+    5281 : (  # Coadjunct Scoped Remote Sensor Booster
         7218,  # Piercing ECCM Emitter I
     ),
-    5365: (  # Cetus Scoped Burst Jammer
+    5365 : (  # Cetus Scoped Burst Jammer
         5359,  # 1Z-3 Subversive ECM Eruption
     ),
-    1973: (  # Sensor Booster I
+    1973 : (  # Sensor Booster I
         1947,  # ECCM - Radar I
         2002,  # ECCM - Ladar I
         2003,  # ECCM - Magnetometric I
         2004,  # ECCM - Gravimetric I
         2005,  # ECCM - Omni I
     ),
-    1951: (  # 'Basic' Tracking Enhancer
+    1951 : (  # 'Basic' Tracking Enhancer
         6322,  # Beta-Nought Tracking Mode
         6323,  # Azimuth Descalloping Tracking Enhancer
         6324,  # F-AQ Delay-Line Scan Tracking Subroutines
         6321,  # Beam Parallax Tracking Program
     ),
-    521: (  # 'Basic' Damage Control
+    521  : (  # 'Basic' Damage Control
         5829,  # GLFF Containment Field
         5831,  # Interior Force Field Array
         5835,  # F84 Local Damage System
@@ -43,13 +42,13 @@ CONVERSIONS = {
         22939,  # 'Boss' Remote Sensor Booster
         22941,  # 'Entrepreneur' Remote Sensor Booster
     ),
-    5443: (  # Faint Epsilon Scoped Warp Scrambler
+    5443 : (  # Faint Epsilon Scoped Warp Scrambler
         5441,  # Fleeting Progressive Warp Scrambler I
     ),
-    1963: (  # Remote Sensor Booster I
+    1963 : (  # Remote Sensor Booster I
         1959,  # ECCM Projector I
     ),
-    6325: (  # Fourier Compact Tracking Enhancer
+    6325 : (  # Fourier Compact Tracking Enhancer
         6326,  # Sigma-Nought Tracking Mode I
         6327,  # Auto-Gain Control Tracking Enhancer I
         6328,  # F-aQ Phase Code Tracking Subroutines
@@ -69,19 +68,19 @@ CONVERSIONS = {
     22919: (  # 'Monopoly' Magnetic Field Stabilizer
         22917,  # 'Capitalist' Magnetic Field Stabilizer I
     ),
-    5839: (  # IFFA Compact Damage Control
+    5839 : (  # IFFA Compact Damage Control
         5841,  # Emergency Damage Control I
         5843,  # F85 Peripheral Damage System I
         5837,  # Pseudoelectron Containment Field I
     ),
-    522: (  # 'Micro' Cap Battery
+    522  : (  # 'Micro' Cap Battery
         4747,  # Micro Ld-Acid Capacitor Battery I
         4751,  # Micro Ohm Capacitor Reserve I
         4745,  # Micro F-4a Ld-Sulfate Capacitor Charge Unit
         4749,  # Micro Peroxide Capacitor Power Cell
         3480,  # Micro Capacitor Battery II
     ),
-    518: (  # 'Basic' Gyrostabilizer
+    518  : (  # 'Basic' Gyrostabilizer
         5915,  # Lateral Gyrostabilizer
         5919,  # F-M2 Weapon Inertial Suspensor
         5913,  # Hydraulic Stabilization Actuator
@@ -90,19 +89,19 @@ CONVERSIONS = {
     19931: (  # Compulsive Scoped Multispectral ECM
         19933,  # 'Hypnos' Multispectral ECM I
     ),
-    5403: (  # Faint Scoped Warp Disruptor
+    5403 : (  # Faint Scoped Warp Disruptor
         5401,  # Fleeting Warp Disruptor I
     ),
     23902: (  # 'Trebuchet' Heat Sink I
         23900,  # 'Mangonel' Heat Sink I
     ),
-    1893: (  # 'Basic' Heat Sink
+    1893 : (  # 'Basic' Heat Sink
         5845,  # Heat Exhaust System
         5856,  # C3S Convection Thermal Radiator
         5855,  # 'Boreas' Coolant System
         5854,  # Stamped Heat Sink
     ),
-    6160: (  # F-90 Compact Sensor Booster
+    6160 : (  # F-90 Compact Sensor Booster
         20214,  # Extra Radar ECCM Scanning Array I
         20220,  # Extra Ladar ECCM Scanning Array I
         20226,  # Extra Gravimetric ECCM Scanning Array I
@@ -124,40 +123,40 @@ CONVERSIONS = {
     19952: (  # Umbra Scoped Radar ECM
         9520,  # 'Penumbra' White Noise ECM
     ),
-    1952: (  # Sensor Booster II
+    1952 : (  # Sensor Booster II
         2258,  # ECCM - Omni II
         2259,  # ECCM - Gravimetric II
         2260,  # ECCM - Ladar II
         2261,  # ECCM - Magnetometric II
         2262,  # ECCM - Radar II
     ),
-    5282: (  # Linked Enduring Sensor Booster
+    5282 : (  # Linked Enduring Sensor Booster
         7219,  # Scattering ECCM Projector I
     ),
-    1986: (  # Signal Amplifier I
+    1986 : (  # Signal Amplifier I
         2579,  # Gravimetric Backup Array I
         2583,  # Ladar Backup Array I
         2587,  # Magnetometric Backup Array I
         2591,  # Multi Sensor Backup Array I
         4013,  # RADAR Backup Array I
     ),
-    4871: (  # Large Compact Pb-Acid Cap Battery
+    4871 : (  # Large Compact Pb-Acid Cap Battery
         4875,  # Large Ohm Capacitor Reserve I
         4869,  # Large F-4a Ld-Sulfate Capacitor Charge Unit
         4873,  # Large Peroxide Capacitor Power Cell
     ),
-    1964: (  # Remote Sensor Booster II
+    1964 : (  # Remote Sensor Booster II
         1960,  # ECCM Projector II
     ),
-    5933: (  # Counterbalanced Compact Gyrostabilizer
+    5933 : (  # Counterbalanced Compact Gyrostabilizer
         5931,  # Cross-Lateral Gyrostabilizer I
         5935,  # F-M3 Munition Inertial Suspensor
         5929,  # Pneumatic Stabilization Actuator I
     ),
-    4025: (  # X5 Enduring Stasis Webifier
+    4025 : (  # X5 Enduring Stasis Webifier
         4029,  # 'Langour' Drive Disruptor I
     ),
-    4027: (  # Fleeting Compact Stasis Webifier
+    4027 : (  # Fleeting Compact Stasis Webifier
         4031,  # Patterned Stasis Web I
     ),
     22937: (  # 'Enterprise' Remote Tracking Computer
@@ -166,7 +165,7 @@ CONVERSIONS = {
     22929: (  # 'Marketeer' Tracking Computer
         22927,  # 'Economist' Tracking Computer I
     ),
-    1987: (  # Signal Amplifier II
+    1987 : (  # Signal Amplifier II
         2580,  # Gravimetric Backup Array II
         2584,  # Ladar Backup Array II
         2588,  # Magnetometric Backup Array II
@@ -176,13 +175,13 @@ CONVERSIONS = {
     19939: (  # Enfeebling Scoped Ladar ECM
         9522,  # Faint Phase Inversion ECM I
     ),
-    5340: (  # P-S Compact Remote Tracking Computer
+    5340 : (  # P-S Compact Remote Tracking Computer
         5341,  # 'Prayer' Remote Tracking Computer
     ),
     19814: (  # Phased Scoped Target Painter
         19808,  # Partial Weapon Navigation
     ),
-    1949: (  # 'Basic' Signal Amplifier
+    1949 : (  # 'Basic' Signal Amplifier
         1946,  # Basic RADAR Backup Array
         1982,  # Basic Ladar Backup Array
         1983,  # Basic Gravimetric Backup Array
@@ -223,10 +222,10 @@ CONVERSIONS = {
     23416: (  # 'Peace' Large Remote Armor Repairer
         None,  # 'Pacifier' Large Remote Armor Repairer
     ),
-    6176: (  # F-12 Enduring Tracking Computer
+    6176 : (  # F-12 Enduring Tracking Computer
         6174,  # Monopulse Tracking Mechanism I
     ),
-    6159: (  # Alumel-Wired Enduring Sensor Booster
+    6159 : (  # Alumel-Wired Enduring Sensor Booster
         7917,  # Alumel Radar ECCM Sensor Array I
         7918,  # Alumel Ladar ECCM Sensor Array I
         7922,  # Alumel Gravimetric ECCM Sensor Array I
@@ -248,7 +247,7 @@ CONVERSIONS = {
         7914,  # Prototype ECCM Magnetometric Sensor Cluster
         6158,  # Prototype Sensor Booster
     ),
-    5849: (  # Extruded Compact Heat Sink
+    5849 : (  # Extruded Compact Heat Sink
         5846,  # Thermal Exhaust System I
         5858,  # C4S Coiled Circuit Thermal Radiator
         5857,  # 'Skadi' Coolant System I
@@ -264,15 +263,15 @@ CONVERSIONS = {
     22945: (  # 'Executive' Remote Sensor Dampener
         22943,  # 'Broker' Remote Sensor Dampener I
     ),
-    6173: (  # Optical Compact Tracking Computer
+    6173 : (  # Optical Compact Tracking Computer
         6175,  # 'Orion' Tracking CPU I
     ),
-    5279: (  # F-23 Compact Remote Sensor Booster
+    5279 : (  # F-23 Compact Remote Sensor Booster
         7217,  # Spot Pulsing ECCM I
         7220,  # Phased Muon ECCM Caster I
         5280,  # Connected Remote Sensor Booster
     ),
-    4787: (  # Small Compact Pb-Acid Cap Battery
+    4787 : (  # Small Compact Pb-Acid Cap Battery
         4791,  # Small Ohm Capacitor Reserve I
         4785,  # Small F-4a Ld-Sulfate Capacitor Charge Unit
         4789,  # Small Peroxide Capacitor Power Cell
@@ -280,7 +279,7 @@ CONVERSIONS = {
     19946: (  # BZ-5 Scoped Gravimetric ECM
         9519,  # FZ-3 Subversive Spatial Destabilizer ECM
     ),
-    6073: (  # Medium Compact Pb-Acid Cap Battery
+    6073 : (  # Medium Compact Pb-Acid Cap Battery
         6097,  # Medium Ohm Capacitor Reserve I
         6111,  # Medium F-4a Ld-Sulfate Capacitor Charge Unit
         6083,  # Medium Peroxide Capacitor Power Cell
@@ -288,7 +287,7 @@ CONVERSIONS = {
     21484: (  # 'Full Duplex' Ballistic Control System
         21482,  # Ballistic 'Purge' Targeting System I
     ),
-    6296: (  # F-89 Compact Signal Amplifier
+    6296 : (  # F-89 Compact Signal Amplifier
         6218,  # Protected Gravimetric Backup Cluster I
         6222,  # Protected Ladar Backup Cluster I
         6226,  # Protected Magnetometric Backup Cluster I
@@ -325,16 +324,17 @@ CONVERSIONS = {
         6293,  # Wavelength Signal Enhancer I
         6295,  # Type-D Attenuation Signal Augmentation
     ),
-    5302: (  # Phased Muon Scoped Sensor Dampener
+    5302 : (  # Phased Muon Scoped Sensor Dampener
         5300,  # Indirect Scanning Dampening Unit I
     ),
 }
 
+
 def upgrade(saveddata_engine):
-
     # Convert modules
-    for replacement_item, list in CONVERSIONS.iteritems():
+    for replacement_item, list in CONVERSIONS.items():
         for retired_item in list:
-            saveddata_engine.execute('UPDATE "modules" SET "itemID" = ? WHERE "itemID" = ?', (replacement_item, retired_item))
-            saveddata_engine.execute('UPDATE "cargo" SET "itemID" = ? WHERE "itemID" = ?', (replacement_item, retired_item))
-
+            saveddata_engine.execute('UPDATE "modules" SET "itemID" = ? WHERE "itemID" = ?',
+                                     (replacement_item, retired_item))
+            saveddata_engine.execute('UPDATE "cargo" SET "itemID" = ? WHERE "itemID" = ?',
+                                     (replacement_item, retired_item))

eos/db/migrations/upgrade13.py

@@ -6,10 +6,11 @@ Migration 13
 
 import sqlalchemy
 
+
 def upgrade(saveddata_engine):
     # Update fits schema to include implant location attribute
     try:
         saveddata_engine.execute("SELECT implantLocation FROM fits LIMIT 1")
     except sqlalchemy.exc.DatabaseError:
         saveddata_engine.execute("ALTER TABLE fits ADD COLUMN implantLocation INTEGER;")
-        saveddata_engine.execute("UPDATE fits SET implantLocation = 0")
+        saveddata_engine.execute("UPDATE fits SET implantLocation = 0")

eos/db/migrations/upgrade14.py

@@ -6,8 +6,10 @@ Migration 14
 
 import sqlalchemy
 
+
 def upgrade(saveddata_engine):
-    if saveddata_engine.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='fighters'").scalar() == 'fighters':
+    if saveddata_engine.execute(
+            "SELECT name FROM sqlite_master WHERE type='table' AND name='fighters'").scalar() == 'fighters':
         # Fighters table exists
         try:
             saveddata_engine.execute("SELECT active FROM fighters LIMIT 1")
@@ -16,4 +18,4 @@ def upgrade(saveddata_engine):
             # (they will be recreated)
 
             saveddata_engine.execute("DROP TABLE fighters")
-            saveddata_engine.execute("DROP TABLE fightersAbilities")
+            saveddata_engine.execute("DROP TABLE fightersAbilities")

eos/db/migrations/upgrade15.py

@@ -4,10 +4,8 @@ Migration 15
 - Delete projected modules on citadels
 """
 
-import sqlalchemy
 
 def upgrade(saveddata_engine):
-
     sql = """
     DELETE FROM modules WHERE ID IN
     (

eos/db/migrations/upgrade16.py

@@ -6,6 +6,7 @@ Migration 16
 
 import sqlalchemy
 
+
 def upgrade(saveddata_engine):
     # Update fits schema to include notes attribute
     try:

eos/db/migrations/upgrade17.py

@@ -4,8 +4,6 @@ Migration 17
 - Moves all fleet boosters to the new schema
 """
 
-import sqlalchemy
-
 
 def upgrade(saveddata_engine):
     from eos.db import saveddata_session
@@ -18,22 +16,31 @@ def upgrade(saveddata_engine):
           JOIN wings w on w.ID = s.wingID
           JOIN gangs g on g.ID = w.gangID
           """
+    try:
+        results = saveddata_session.execute(sql)
 
-    results = saveddata_session.execute(sql)
+        inserts = []
 
-    inserts = []
+        for row in results:
+            boosted = row["boostedFit"]
+            types = ("squad", "wing", "gang")
+            for x in types:
+                value = row["{}Boost".format(x)]
+                if value is None:
+                    continue
 
-    for row in results:
-        boosted = row["boostedFit"]
-        types = ("squad", "wing", "gang")
-        for x in types:
-            value = row["{}Boost".format(x)]
-            if value is None:
-                continue
-
-            inserts.append({"boosterID": value, "boostedID": boosted, "active": 1})
-            try:
-                saveddata_session.execute(commandFits_table.insert(), {"boosterID": value, "boostedID": boosted, "active": 1})
-            except Exception, e:
-                pass
-    saveddata_session.commit()
+                inserts.append({"boosterID": value, "boostedID": boosted, "active": 1})
+                try:
+                    saveddata_session.execute(commandFits_table.insert(),
+                                              {"boosterID": value, "boostedID": boosted, "active": 1})
+                except (KeyboardInterrupt, SystemExit):
+                    raise
+                except Exception:
+                    pass
+        saveddata_session.commit()
+    except (KeyboardInterrupt, SystemExit):
+        raise
+    except:
+        # Shouldn't fail unless you have updated database without the old fleet schema and manually modify the database version
+        # If it does, simply fail. Fleet data migration isn't critically important here
+        pass

eos/db/migrations/upgrade18.py

@@ -4,27 +4,26 @@ Migration 8
 - Converts modules from old Warfare Links to Command Modules
 """
 
-
 CONVERSIONS = {
     42526: (  # Armor Command Burst I
-        20069, # Armored Warfare Link - Damage Control I
-        20409, # Armored Warfare Link - Passive Defense I
-        22227, # Armored Warfare Link - Rapid Repair I
+        20069,  # Armored Warfare Link - Damage Control I
+        20409,  # Armored Warfare Link - Passive Defense I
+        22227,  # Armored Warfare Link - Rapid Repair I
     ),
     43552: (  # Armor Command Burst II
-        4264, # Armored Warfare Link - Damage Control II
-        4266, # Armored Warfare Link - Passive Defense II
-        4266, # Armored Warfare Link - Rapid Repair II
+        4264,  # Armored Warfare Link - Damage Control II
+        4266,  # Armored Warfare Link - Passive Defense II
+        4266,  # Armored Warfare Link - Rapid Repair II
     ),
     42527: (  # Information Command Burst I
-        11052, # Information Warfare Link - Sensor Integrity I
-        20405, # Information Warfare Link - Recon Operation I
-        20406, # Information Warfare Link - Electronic Superiority I
+        11052,  # Information Warfare Link - Sensor Integrity I
+        20405,  # Information Warfare Link - Recon Operation I
+        20406,  # Information Warfare Link - Electronic Superiority I
     ),
     43554: (  # Information Command Burst II
-        4268, # Information Warfare Link - Electronic Superiority II
-        4270, # Information Warfare Link - Recon Operation II
-        4272, # Information Warfare Link - Sensor Integrity II
+        4268,  # Information Warfare Link - Electronic Superiority II
+        4270,  # Information Warfare Link - Recon Operation II
+        4272,  # Information Warfare Link - Sensor Integrity II
     ),
     42529: (  # Shield Command Burst I
         20124,  # Siege Warfare Link - Active Shielding I
@@ -34,17 +33,17 @@ CONVERSIONS = {
     43555: (  # Shield Command Burst II
         4280,  # Siege Warfare Link - Active Shielding II
         4282,  # Siege Warfare Link - Shield Efficiency II
-        4284   # Siege Warfare Link - Shield Harmonizing II
+        4284  # Siege Warfare Link - Shield Harmonizing II
     ),
     42530: (  # Skirmish Command Burst I
-        11017, # Skirmish Warfare Link - Interdiction Maneuvers I
-        20070, # Skirmish Warfare Link - Evasive Maneuvers I
-        20408, # Skirmish Warfare Link - Rapid Deployment I
+        11017,  # Skirmish Warfare Link - Interdiction Maneuvers I
+        20070,  # Skirmish Warfare Link - Evasive Maneuvers I
+        20408,  # Skirmish Warfare Link - Rapid Deployment I
     ),
     43556: (  # Skirmish Command Burst II
         4286,  # Skirmish Warfare Link - Evasive Maneuvers II
         4288,  # Skirmish Warfare Link - Interdiction Maneuvers II
-        4290   # Skirmish Warfare Link - Rapid Deployment II
+        4290  # Skirmish Warfare Link - Rapid Deployment II
     ),
     42528: (  # Mining Foreman Burst I
         22553,  # Mining Foreman Link - Harvester Capacitor Efficiency I
@@ -54,15 +53,16 @@ CONVERSIONS = {
     43551: (  # Mining Foreman Burst II
         4274,  # Mining Foreman Link - Harvester Capacitor Efficiency II
         4276,  # Mining Foreman Link - Laser Optimization II
-        4278   # Mining Foreman Link - Mining Laser Field Enhancement II
+        4278  # Mining Foreman Link - Mining Laser Field Enhancement II
     ),
 }
 
+
 def upgrade(saveddata_engine):
-
     # Convert modules
-    for replacement_item, list in CONVERSIONS.iteritems():
+    for replacement_item, list in CONVERSIONS.items():
         for retired_item in list:
-            saveddata_engine.execute('UPDATE "modules" SET "itemID" = ? WHERE "itemID" = ?', (replacement_item, retired_item))
-            saveddata_engine.execute('UPDATE "cargo" SET "itemID" = ? WHERE "itemID" = ?', (replacement_item, retired_item))
-
+            saveddata_engine.execute('UPDATE "modules" SET "itemID" = ? WHERE "itemID" = ?',
+                                     (replacement_item, retired_item))
+            saveddata_engine.execute('UPDATE "cargo" SET "itemID" = ? WHERE "itemID" = ?',
+                                     (replacement_item, retired_item))

eos/db/migrations/upgrade19.py

@@ -4,17 +4,15 @@ Migration 19
 - Deletes broken references to fits from the commandFits table (see GH issue #844)
 """
 
-import sqlalchemy
-
 
 def upgrade(saveddata_engine):
     from eos.db import saveddata_session
 
     sql = """
-          DELETE FROM commandFits
-          WHERE boosterID NOT IN (select ID from fits)
-	      OR boostedID NOT IN (select ID from fits)
-          """
+        DELETE FROM commandFits
+        WHERE boosterID NOT IN (select ID from fits)
+        OR boostedID NOT IN (select ID from fits)
+        """
 
     saveddata_session.execute(sql)
     saveddata_session.commit()

eos/db/migrations/upgrade2.py

@@ -6,6 +6,7 @@ Migration 2
 
 import sqlalchemy
 
+
 def upgrade(saveddata_engine):
     # Update characters schema to include default chars
     try:

eos/db/migrations/upgrade20.py

@@ -0,0 +1,15 @@
+"""
+Migration 20
+
+- Adds support for alpha clones to the characters table
+"""
+
+import sqlalchemy
+
+
+def upgrade(saveddata_engine):
+    # Update characters schema to include alphaCloneID
+    try:
+        saveddata_engine.execute("SELECT alphaCloneID FROM characters LIMIT 1")
+    except sqlalchemy.exc.DatabaseError:
+        saveddata_engine.execute("ALTER TABLE characters ADD COLUMN alphaCloneID INTEGER;")

eos/db/migrations/upgrade21.py

@@ -0,0 +1,10 @@
+"""
+Migration 21
+
+- Fixes discrepancy in drone table where we may have an amount active that is not equal to the amount in the stack
+  (we don't support activating only 2/5 drones). See GH issue #728
+"""
+
+
+def upgrade(saveddata_engine):
+    saveddata_engine.execute("UPDATE drones SET amountActive = amount where amountActive > 0 AND amountActive <> amount;")

eos/db/migrations/upgrade22.py

@@ -0,0 +1,45 @@
+"""
+Migration 22
+
+- Adds the created and modified fields to most tables
+"""
+import sqlalchemy
+
+
+def upgrade(saveddata_engine):
+
+    # 1 = created only
+    # 2 = created and modified
+    tables = {
+        "boosters": 2,
+        "cargo": 2,
+        "characters": 2,
+        # "crest": 1,
+        "damagePatterns": 2,
+        "drones": 2,
+        "fighters": 2,
+        "fits": 2,
+        "projectedFits": 2,
+        "commandFits": 2,
+        "implants": 2,
+        "implantSets": 2,
+        "modules": 2,
+        "overrides": 2,
+        "characterSkills": 2,
+        "targetResists": 2
+    }
+
+    for table in list(tables.keys()):
+
+        # midnight brain, there's probably a much more simple way to do this, but fuck it
+        if tables[table] > 0:
+            try:
+                saveddata_engine.execute("SELECT created FROM {0} LIMIT 1;".format(table))
+            except sqlalchemy.exc.DatabaseError:
+                saveddata_engine.execute("ALTER TABLE {} ADD COLUMN created DATETIME;".format(table))
+
+        if tables[table] > 1:
+            try:
+                saveddata_engine.execute("SELECT modified FROM {0} LIMIT 1;".format(table))
+            except sqlalchemy.exc.DatabaseError:
+                saveddata_engine.execute("ALTER TABLE {} ADD COLUMN modified DATETIME;".format(table))

eos/db/migrations/upgrade23.py

@@ -0,0 +1,13 @@
+"""
+Migration 23
+
+- Adds a sec status field to the character table
+"""
+import sqlalchemy
+
+
+def upgrade(saveddata_engine):
+    try:
+        saveddata_engine.execute("SELECT secStatus FROM characters LIMIT 1")
+    except sqlalchemy.exc.DatabaseError:
+        saveddata_engine.execute("ALTER TABLE characters ADD COLUMN secStatus FLOAT;")

eos/db/migrations/upgrade24.py

@@ -0,0 +1,14 @@
+"""
+Migration 24
+
+- Adds a boolean value to fit to signify if fit should ignore restrictions
+"""
+import sqlalchemy
+
+
+def upgrade(saveddata_engine):
+    try:
+        saveddata_engine.execute("SELECT ignoreRestrictions FROM fits LIMIT 1")
+    except sqlalchemy.exc.DatabaseError:
+        saveddata_engine.execute("ALTER TABLE fits ADD COLUMN ignoreRestrictions BOOLEAN")
+        saveddata_engine.execute("UPDATE fits SET ignoreRestrictions = 0")

eos/db/migrations/upgrade26.py

@@ -0,0 +1,9 @@
+"""
+Migration 26
+
+- Deletes invalid command fit relationships caused by a bug (see #1244)
+"""
+
+
+def upgrade(saveddata_engine):
+    saveddata_engine.execute("DELETE FROM commandFits WHERE boosterID NOT IN (SELECT ID FROM fits) OR boostedID NOT IN (SELECT ID FROM fits)")

eos/db/migrations/upgrade27.py

@@ -0,0 +1,9 @@
+"""
+Migration 27
+
+- Resets all alpha clones to 1 (CCP consolidated all alpha's into one skillset)
+"""
+
+
+def upgrade(saveddata_engine):
+    saveddata_engine.execute("UPDATE characters SET alphaCloneID = 1 WHERE alphaCloneID IS NOT NULL")

eos/db/migrations/upgrade28.py

@@ -0,0 +1,18 @@
+"""
+Migration 28
+
+- adds baseItemID and mutaplasmidID to modules table
+"""
+import sqlalchemy
+
+
+def upgrade(saveddata_engine):
+    try:
+        saveddata_engine.execute("SELECT baseItemID FROM modules LIMIT 1")
+    except sqlalchemy.exc.DatabaseError:
+        saveddata_engine.execute("ALTER TABLE modules ADD COLUMN baseItemID INT;")
+
+    try:
+        saveddata_engine.execute("SELECT mutaplasmidID FROM modules LIMIT 1")
+    except sqlalchemy.exc.DatabaseError:
+        saveddata_engine.execute("ALTER TABLE modules ADD COLUMN mutaplasmidID INT;")

eos/db/migrations/upgrade29.py

@@ -0,0 +1,18 @@
+"""
+Migration 29
+
+- adds spoolType and spoolAmount to modules table
+"""
+import sqlalchemy
+
+
+def upgrade(saveddata_engine):
+    try:
+        saveddata_engine.execute("SELECT spoolType FROM modules LIMIT 1")
+    except sqlalchemy.exc.DatabaseError:
+        saveddata_engine.execute("ALTER TABLE modules ADD COLUMN spoolType INT;")
+
+    try:
+        saveddata_engine.execute("SELECT spoolAmount FROM modules LIMIT 1")
+    except sqlalchemy.exc.DatabaseError:
+        saveddata_engine.execute("ALTER TABLE modules ADD COLUMN spoolAmount FLOAT;")

eos/db/migrations/upgrade3.py

@@ -6,6 +6,7 @@ Migration 3
 
 import sqlalchemy
 
+
 def upgrade(saveddata_engine):
     try:
         saveddata_engine.execute("SELECT modeID FROM fits LIMIT 1")

eos/db/migrations/upgrade30.py

@@ -0,0 +1,17 @@
+"""
+Migration 30
+
+- changes to prices table
+"""
+
+
+import sqlalchemy
+
+
+def upgrade(saveddata_engine):
+    try:
+        saveddata_engine.execute("SELECT status FROM prices LIMIT 1")
+    except sqlalchemy.exc.DatabaseError:
+        # Just drop table, table will be re-created by sqlalchemy and
+        # data will be re-fetched
+        saveddata_engine.execute("DROP TABLE prices;")

eos/db/migrations/upgrade31.py

@@ -0,0 +1,15 @@
+"""
+Migration 31
+
+- added fit system security column
+"""
+
+
+import sqlalchemy
+
+
+def upgrade(saveddata_engine):
+    try:
+        saveddata_engine.execute("SELECT systemSecurity FROM fits LIMIT 1")
+    except sqlalchemy.exc.DatabaseError:
+        saveddata_engine.execute("ALTER TABLE fits ADD COLUMN systemSecurity INT")

eos/db/migrations/upgrade32.py

@@ -0,0 +1,16 @@
+"""
+Migration 32
+
+- added speed, sig and radius columns to targetResists table
+"""
+
+
+import sqlalchemy
+
+
+def upgrade(saveddata_engine):
+    for column in ('maxVelocity', 'signatureRadius', 'radius'):
+        try:
+            saveddata_engine.execute("SELECT {} FROM targetResists LIMIT 1;".format(column))
+        except sqlalchemy.exc.DatabaseError:
+            saveddata_engine.execute("ALTER TABLE targetResists ADD COLUMN {} FLOAT;".format(column))

eos/db/migrations/upgrade33.py

@@ -0,0 +1,30 @@
+"""
+Migration 33
+
+Allow use of floats in damage pattern values
+"""
+
+tmpTable = """
+CREATE TABLE "damagePatternsTemp" (
+	"ID" INTEGER NOT NULL,
+	"name" VARCHAR,
+	"emAmount" FLOAT,
+	"thermalAmount" FLOAT,
+	"kineticAmount" FLOAT,
+	"explosiveAmount" FLOAT,
+	"ownerID" INTEGER,
+	"created" DATETIME,
+	"modified" DATETIME,
+	PRIMARY KEY ("ID"),
+	FOREIGN KEY("ownerID") REFERENCES users ("ID")
+)
+"""
+
+
+def upgrade(saveddata_engine):
+    saveddata_engine.execute(tmpTable)
+    saveddata_engine.execute(
+            'INSERT INTO damagePatternsTemp (ID, name, emAmount, thermalAmount, kineticAmount, explosiveAmount, ownerID, created, modified) '
+            'SELECT ID, name, emAmount, thermalAmount, kineticAmount, explosiveAmount, ownerID, created, modified FROM damagePatterns')
+    saveddata_engine.execute('DROP TABLE damagePatterns')
+    saveddata_engine.execute('ALTER TABLE damagePatternsTemp RENAME TO damagePatterns')

eos/db/migrations/upgrade34.py

@@ -0,0 +1,25 @@
+"""
+Migration 34
+
+- Adds projection range columns to projectable entities
+"""
+import sqlalchemy
+
+
+def upgrade(saveddata_engine):
+    try:
+        saveddata_engine.execute("SELECT projectionRange FROM projectedFits LIMIT 1")
+    except sqlalchemy.exc.DatabaseError:
+        saveddata_engine.execute("ALTER TABLE projectedFits ADD COLUMN projectionRange FLOAT;")
+    try:
+        saveddata_engine.execute("SELECT projectionRange FROM modules LIMIT 1")
+    except sqlalchemy.exc.DatabaseError:
+        saveddata_engine.execute("ALTER TABLE modules ADD COLUMN projectionRange FLOAT;")
+    try:
+        saveddata_engine.execute("SELECT projectionRange FROM drones LIMIT 1")
+    except sqlalchemy.exc.DatabaseError:
+        saveddata_engine.execute("ALTER TABLE drones ADD COLUMN projectionRange FLOAT;")
+    try:
+        saveddata_engine.execute("SELECT projectionRange FROM fighters LIMIT 1")
+    except sqlalchemy.exc.DatabaseError:
+        saveddata_engine.execute("ALTER TABLE fighters ADD COLUMN projectionRange FLOAT;")

eos/db/migrations/upgrade35.py

@@ -0,0 +1,166 @@
+"""
+Migration 35
+
+- Remove builtin damage patterns and target profiles from the database
+"""
+
+import sqlalchemy
+
+
+dmgPatterns = (
+    'Uniform',
+    '[Bombs]Concussion Bomb',
+    '[Bombs]Electron Bomb',
+    '[Bombs]Scorch Bomb',
+    '[Bombs]Shrapnel Bomb',
+    '[Exotic Plasma]Baryon',
+    '[Exotic Plasma]Meson',
+    '[Exotic Plasma]Tetryon',
+    '[Exotic Plasma][T2] Mystic',
+    '[Exotic Plasma][T2] Occult',
+    '[Frequency Crystals]Gamma',
+    '[Frequency Crystals]Infrared',
+    '[Frequency Crystals]Microwave',
+    '[Frequency Crystals]Multifrequency',
+    '[Frequency Crystals]Radio',
+    '[Frequency Crystals]Standard',
+    '[Frequency Crystals]Ultraviolet',
+    '[Frequency Crystals]Xray',
+    '[Frequency Crystals][T2] Aurora',
+    '[Frequency Crystals][T2] Conflagration',
+    '[Frequency Crystals][T2] Gleam',
+    '[Frequency Crystals][T2] Scorch',
+    '[Generic]EM',
+    '[Generic]Explosive',
+    '[Generic]Kinetic',
+    '[Generic]Thermal',
+    '[Hybrid Charges]Antimatter',
+    '[Hybrid Charges]Iridium',
+    '[Hybrid Charges]Iron',
+    '[Hybrid Charges]Lead',
+    '[Hybrid Charges]Plutonium',
+    '[Hybrid Charges]Thorium',
+    '[Hybrid Charges]Tungsten',
+    '[Hybrid Charges]Uranium',
+    '[Hybrid Charges][T2] Javelin',
+    '[Hybrid Charges][T2] Null',
+    '[Hybrid Charges][T2] Spike',
+    '[Hybrid Charges][T2] Void',
+    '[Missiles]Inferno',
+    '[Missiles]Mjolnir',
+    '[Missiles]Nova',
+    '[Missiles]Scourge',
+    '[Missiles][Structure) Standup Missile',
+    '[Missiles][Structure] Standup Missile',
+    '[NPC][Asteroid] Angel Cartel',
+    '[NPC][Asteroid] Blood Raiders',
+    '[NPC][Asteroid] Guristas',
+    '[NPC][Asteroid] Rogue Drone',
+    '[NPC][Asteroid] Sanshas Nation',
+    '[NPC][Asteroid] Serpentis',
+    '[NPC][Burner] Ashimmu (Blood Raiders)',
+    '[NPC][Burner] Cruor (Blood Raiders)',
+    '[NPC][Burner] Daredevil (Serpentis)',
+    '[NPC][Burner] Dramiel (Angel)',
+    '[NPC][Burner] Enyo',
+    '[NPC][Burner] Hawk',
+    '[NPC][Burner] Jaguar',
+    '[NPC][Burner] Sentinel',
+    '[NPC][Burner] Succubus (Sanshas Nation)',
+    '[NPC][Burner] Talos',
+    '[NPC][Burner] Vengeance',
+    '[NPC][Burner] Worm (Guristas)',
+    '[NPC][Deadspace] Angel Cartel',
+    '[NPC][Deadspace] Blood Raiders',
+    '[NPC][Deadspace] Guristas',
+    '[NPC][Deadspace] Rogue Drone',
+    '[NPC][Deadspace] Sanshas Nation',
+    '[NPC][Deadspace] Serpentis',
+    '[NPC][Mission] Amarr Empire',
+    '[NPC][Mission] CONCORD',
+    '[NPC][Mission] Caldari State',
+    '[NPC][Mission] Gallente Federation',
+    '[NPC][Mission] Khanid',
+    '[NPC][Mission] Minmatar Republic',
+    '[NPC][Mission] Mordus Legion',
+    '[NPC][Mission] Thukker',
+    '[NPC][Other] Sansha Incursion',
+    '[NPC][Other] Sleepers',
+    '[Projectile Ammo]Carbonized Lead',
+    '[Projectile Ammo]Depleted Uranium',
+    '[Projectile Ammo]EMP',
+    '[Projectile Ammo]Fusion',
+    '[Projectile Ammo]Nuclear',
+    '[Projectile Ammo]Phased Plasma',
+    '[Projectile Ammo]Proton',
+    '[Projectile Ammo]Titanium Sabot',
+    '[Projectile Ammo][T2] Barrage',
+    '[Projectile Ammo][T2] Hail',
+    '[Projectile Ammo][T2] Quake',
+    '[Projectile Ammo][T2] Tremor')
+
+tgtProfiles = (
+    'Uniform (25%)',
+    'Uniform (50%)',
+    'Uniform (75%)',
+    'Uniform (90%)',
+    '[NPC][Asteroid] Angel Cartel',
+    '[NPC][Asteroid] Blood Raiders',
+    '[NPC][Asteroid] Guristas',
+    '[NPC][Asteroid] Rogue Drones',
+    '[NPC][Asteroid] Sanshas Nation',
+    '[NPC][Asteroid] Serpentis',
+    '[NPC][Burner] Ashimmu (Blood Raiders)',
+    '[NPC][Burner] Cruor (Blood Raiders)',
+    '[NPC][Burner] Daredevil (Serpentis)',
+    '[NPC][Burner] Dramiel (Angel)',
+    '[NPC][Burner] Enyo',
+    '[NPC][Burner] Hawk',
+    '[NPC][Burner] Jaguar',
+    '[NPC][Burner] Sentinel',
+    '[NPC][Burner] Succubus (Sanshas Nation)',
+    '[NPC][Burner] Talos',
+    '[NPC][Burner] Vengeance',
+    '[NPC][Burner] Worm (Guristas)',
+    '[NPC][Deadspace] Angel Cartel',
+    '[NPC][Deadspace] Blood Raiders',
+    '[NPC][Deadspace] Guristas',
+    '[NPC][Deadspace] Rogue Drones',
+    '[NPC][Deadspace] Sanshas Nation',
+    '[NPC][Deadspace] Serpentis',
+    '[NPC][Mission] Amarr Empire',
+    '[NPC][Mission] CONCORD',
+    '[NPC][Mission] Caldari State',
+    '[NPC][Mission] Gallente Federation',
+    '[NPC][Mission] Khanid',
+    '[NPC][Mission] Minmatar Republic',
+    '[NPC][Mission] Mordus Legion',
+    '[NPC][Other] Sansha Incursion',
+    '[NPC][Other] Sleeper',
+    '[T1 Resist]Armor',
+    '[T1 Resist]Armor (+T2 DCU)',
+    '[T1 Resist]Hull',
+    '[T1 Resist]Hull (+T2 DCU)',
+    '[T1 Resist]Shield',
+    '[T1 Resist]Shield (+T2 DCU)',
+    '[T2 Resist]Amarr (Armor)',
+    '[T2 Resist]Amarr (Shield)',
+    '[T2 Resist]Caldari (Armor)',
+    '[T2 Resist]Caldari (Shield)',
+    '[T2 Resist]Gallente (Armor)',
+    '[T2 Resist]Gallente (Shield)',
+    '[T2 Resist]Minmatar (Armor)',
+    '[T2 Resist]Minmatar (Shield)')
+
+
+def upgrade(saveddata_engine):
+    saveddata_engine.execute('DELETE FROM damagePatterns WHERE name in ({});'.format(', '.join('\'{}\''.format(n) for n in dmgPatterns)))
+    saveddata_engine.execute('DELETE FROM targetResists WHERE name in ({});'.format(', '.join('\'{}\''.format(n) for n in tgtProfiles)))
+    try:
+        saveddata_engine.execute("SELECT builtinDamagePatternID FROM fits LIMIT 1")
+    except sqlalchemy.exc.DatabaseError:
+        saveddata_engine.execute("ALTER TABLE fits ADD COLUMN builtinDamagePatternID INT;")
+    try:
+        saveddata_engine.execute("SELECT builtinTargetResistsID FROM fits LIMIT 1")
+    except sqlalchemy.exc.DatabaseError:
+        saveddata_engine.execute("ALTER TABLE fits ADD COLUMN builtinTargetResistsID INT;")

eos/db/migrations/upgrade36.py

@@ -0,0 +1,84 @@
+"""
+Migration 36
+
+- Shield Booster, Armor Repairer and Capacitor Transfer tiericide
+"""
+
+CONVERSIONS = {
+    6441: (  # Small Clarity Ward Enduring Shield Booster
+        6443,  # Small Converse Deflection Catalyzer
+    ),
+    6437: (  # Small C5-L Compact Shield Booster
+        6439,  # Small Neutron Saturation Injector I
+    ),
+    10868: (  # Medium Clarity Ward Enduring Shield Booster
+        10870,  # Medium Converse Deflection Catalyzer
+    ),
+    10872: (  # Medium C5-L Compact Shield Booster
+        10866,  # Medium Neutron Saturation Injector I
+    ),
+    10876: (  # Large Clarity Ward Enduring Shield Booster
+        10878,  # Large Converse Deflection Catalyzer
+    ),
+    10880: (  # Large C5-L Compact Shield Booster
+        10874,  # Large Neutron Saturation Injector I
+    ),
+    10884: (  # X-Large Clarity Ward Enduring Shield Booster
+        10886,  # X-Large Converse Deflection Catalyzer
+    ),
+    10888: (  # X-Large C5-L Compact Shield Booster
+        10882,  # X-Large Neutron Saturation Injector I
+    ),
+    4533: (  # Small ACM Compact Armor Repairer
+        4531,  # Small Inefficient Armor Repair Unit
+    ),
+    4529: (  # Small I-a Enduring Armor Repairer
+        4535,  # Small Automated Carapace Restoration
+    ),
+    4573: (  # Medium ACM Compact Armor Repairer
+        4571,  # Medium Inefficient Armor Repair Unit
+    ),
+    4569: (  # Medium I-a Enduring Armor Repairer
+        4575,  # Medium Automated Carapace Restoration
+    ),
+    22889: (  # 'Meditation' Medium Armor Repairer I
+        4579,  # Medium Nano Armor Repair Unit I
+    ),
+    4613: (  # Large ACM Compact Armor Repairer
+        4611,  # Large Inefficient Armor Repair Unit
+    ),
+    4609: (  # Large I-a Enduring Armor Repairer
+        4615,  # Large Automated Carapace Restoration
+    ),
+    22891: (  # 'Protest' Large Armor Repairer I
+        4621,  # Large 'Reprieve' Vestment Reconstructer I
+    ),
+    5093: (  # Small Radiative Scoped Remote Capacitor Transmitter
+        5087,  # Small Partial E95a Remote Capacitor Transmitter
+    ),
+    5091: (  # Small Inductive Compact Remote Capacitor Transmitter
+        5089,  # Small Murky Remote Capacitor Transmitter
+    ),
+    16489: (  # Medium Radiative Scoped Remote Capacitor Transmitter
+        16493,  # Medium Partial E95b Remote Capacitor Transmitter
+    ),
+    16495: (  # Medium Inductive Compact Remote Capacitor Transmitter
+        16491,  # Medium Murky Remote Capacitor Transmitter
+    ),
+    16481: (  # Large Radiative Scoped Remote Capacitor Transmitter
+        16485,  # Large Partial E95c Remote Capacitor Transmitter
+    ),
+    16487: (  # Large Inductive Compact Remote Capacitor Transmitter
+        16483,  # Large Murky Remote Capacitor Transmitter
+    )
+}
+
+
+def upgrade(saveddata_engine):
+    # Convert modules
+    for replacement_item, list in CONVERSIONS.items():
+        for retired_item in list:
+            saveddata_engine.execute('UPDATE "modules" SET "itemID" = ? WHERE "itemID" = ?',
+                                     (replacement_item, retired_item))
+            saveddata_engine.execute('UPDATE "cargo" SET "itemID" = ? WHERE "itemID" = ?',
+                                     (replacement_item, retired_item))

eos/db/migrations/upgrade37.py

@@ -0,0 +1,44 @@
+"""
+Migration 37
+
+- Capacitor Booster tiericide
+"""
+
+CONVERSIONS = {
+    4959: (  # 'Seed' Micro Capacitor Booster I
+        4957,  # Micro Brief Capacitor Overcharge I
+        4961,  # Micro Tapered Capacitor Infusion I
+        4955,  # Micro F-RX Prototype Capacitor Boost
+        3556,  # Micro Capacitor Booster I
+        3558,  # Micro Capacitor Booster II
+        15774,  # Ammatar Navy Micro Capacitor Booster
+        14180,  # Dark Blood Micro Capacitor Booster
+        14182,  # True Sansha Micro Capacitor Booster
+        15782,  # Imperial Navy Micro Capacitor Booster
+    ),
+    5011: (  # Small F-RX Compact Capacitor Booster
+        5009,  # Small Brief Capacitor Overcharge I
+        5013,  # Small Tapered Capacitor Infusion I
+        5007,  # Small F-RX Prototype Capacitor Boost
+    ),
+    4833: (  # Medium F-RX Compact Capacitor Booster
+        4831,  # Medium Brief Capacitor Overcharge I
+        4835,  # Medium Tapered Capacitor Infusion I
+        4829,  # Medium F-RX Prototype Capacitor Boost
+    ),
+    5051: (  # Heavy F-RX Compact Capacitor Booster
+        5049,  # Heavy Brief Capacitor Overcharge I
+        5053,  # Heavy Tapered Capacitor Infusion I
+        5047,  # Heavy F-RX Prototype Capacitor Boost
+    )
+}
+
+
+def upgrade(saveddata_engine):
+    # Convert modules
+    for replacement_item, list in CONVERSIONS.items():
+        for retired_item in list:
+            saveddata_engine.execute('UPDATE "modules" SET "itemID" = ? WHERE "itemID" = ?',
+                                     (replacement_item, retired_item))
+            saveddata_engine.execute('UPDATE "cargo" SET "itemID" = ? WHERE "itemID" = ?',
+                                     (replacement_item, retired_item))

eos/db/migrations/upgrade38.py

@@ -0,0 +1,42 @@
+"""
+Migration 38
+
+- Armor hardener tiericide
+"""
+
+CONVERSIONS = {
+    16357: (  # Experimental Enduring EM Armor Hardener I
+        16353,  # Upgraded Armor EM Hardener I
+    ),
+    16365: (  # Experimental Enduring Explosive Armor Hardener I
+        16361,  # Upgraded Armor Explosive Hardener I
+    ),
+    16373: (  # Experimental Enduring Kinetic Armor Hardener I
+        16369,  # Upgraded Armor Kinetic Hardener I
+    ),
+    16381: (  # Experimental Enduring Thermal Armor Hardener I
+        16377,  # Upgraded Armor Thermal Hardener I
+    ),
+    16359: (  # Prototype Compact EM Armor Hardener I
+        16355,  # Limited Armor EM Hardener I
+    ),
+    16367: (  # Prototype Compact Explosive Armor Hardener I
+        16363,  # Limited Armor Explosive Hardener I
+    ),
+    16375: (  # Prototype Compact Kinetic Armor Hardener I
+        16371,  # Limited Armor Kinetic Hardener I
+    ),
+    16383: (  # Prototype Compact Thermal Armor Hardener I
+        16379,  # Limited Armor Thermal Hardener I
+    )
+}
+
+
+def upgrade(saveddata_engine):
+    # Convert modules
+    for replacement_item, list in CONVERSIONS.items():
+        for retired_item in list:
+            saveddata_engine.execute('UPDATE "modules" SET "itemID" = ? WHERE "itemID" = ?',
+                                     (replacement_item, retired_item))
+            saveddata_engine.execute('UPDATE "cargo" SET "itemID" = ? WHERE "itemID" = ?',
+                                     (replacement_item, retired_item))

eos/db/migrations/upgrade39.py

@@ -0,0 +1,34 @@
+"""
+Migration 39
+
+- Shield amplifier tiericide
+- CCP getting rid of DB TDs due to exploits
+"""
+
+CONVERSIONS = {
+    1798: (  # 'Basic' EM Shield Amplifier
+        9562,  # Supplemental EM Ward Amplifier
+    ),
+    1804: (  # 'Basic' Explosive Shield Amplifier
+        9574,  # Supplemental Explosive Deflection Amplifier
+    ),
+    1802: (  # 'Basic' Kinetic Shield Amplifier
+        9570,  # Supplemental Kinetic Deflection Amplifier
+    ),
+    1800: (  # 'Basic' Thermal Shield Amplifier
+        9566,  # Supplemental Thermal Dissipation Amplifier
+    ),
+    22933: (  # 'Investor' Tracking Disruptor I
+        32416,  # Dark Blood Tracking Disruptor
+    )
+}
+
+
+def upgrade(saveddata_engine):
+    # Convert modules
+    for replacement_item, list in CONVERSIONS.items():
+        for retired_item in list:
+            saveddata_engine.execute('UPDATE "modules" SET "itemID" = ? WHERE "itemID" = ?',
+                                     (replacement_item, retired_item))
+            saveddata_engine.execute('UPDATE "cargo" SET "itemID" = ? WHERE "itemID" = ?',
+                                     (replacement_item, retired_item))

eos/db/migrations/upgrade4.py

@@ -6,70 +6,69 @@ Migration 4
     from database), which causes pyfa to crash. We therefore replace these
     modules with their new replacements
 
-    Based on http://community.eveonline.com/news/patch-notes/patch-notes-for-proteus/
+    Based on https://www.eveonline.com/news/view/patch-notes-for-proteus
     and output of itemDiff.py
 """
 
-
 CONVERSIONS = {
-    506: (  # 'Basic' Capacitor Power Relay
+    506  : (  # 'Basic' Capacitor Power Relay
         8205,  # Alpha Reactor Control: Capacitor Power Relay
         8209,  # Marked Generator Refitting: Capacitor Power Relay
         8203,  # Partial Power Plant Manager: Capacity Power Relay
         8207,  # Type-E Power Core Modification: Capacitor Power Relay
     ),
-    8177: (  # Mark I Compact Capacitor Power Relay
+    8177 : (  # Mark I Compact Capacitor Power Relay
         8173,  # Beta Reactor Control: Capacitor Power Relay I
     ),
-    8175: (  # Type-D Restrained Capacitor Power Relay
+    8175 : (  # Type-D Restrained Capacitor Power Relay
         8171,  # Local Power Plant Manager: Capacity Power Relay I
     ),
 
-    421: (  # 'Basic' Capacitor Recharger
+    421  : (  # 'Basic' Capacitor Recharger
         4425,  # AGM Capacitor Charge Array,
         4421,  # F-a10 Buffer Capacitor Regenerator
         4423,  # Industrial Capacitor Recharger
         4427,  # Secondary Parallel Link-Capacitor
     ),
-    4435: (  # Eutectic Compact Cap Recharger
+    4435 : (  # Eutectic Compact Cap Recharger
         4433,  # Barton Reactor Capacitor Recharger I
         4431,  # F-b10 Nominal Capacitor Regenerator
         4437,  # Fixed Parallel Link-Capacitor I
     ),
 
-    1315: (  # 'Basic' Expanded Cargohold
+    1315 : (  # 'Basic' Expanded Cargohold
         5483,  # Alpha Hull Mod Expanded Cargo
         5479,  # Marked Modified SS Expanded Cargo
         5481,  # Partial Hull Conversion Expanded Cargo
         5485,  # Type-E Altered SS Expanded Cargo
     ),
-    5493: (  # Type-D Restrained Expanded Cargo
+    5493 : (  # Type-D Restrained Expanded Cargo
         5491,  # Beta Hull Mod Expanded Cargo
         5489,  # Local Hull Conversion Expanded Cargo I
         5487,  # Mark I Modified SS Expanded Cargo
     ),
 
-    1401: (  # 'Basic' Inertial Stabilizers
+    1401 : (  # 'Basic' Inertial Stabilizers
         5523,  # Alpha Hull Mod Inertial Stabilizers
         5521,  # Partial Hull Conversion Inertial Stabilizers
         5525,  # Type-E Altered SS Inertial Stabilizers
     ),
-    5533: (  # Type-D Restrained Inertial Stabilizers
+    5533 : (  # Type-D Restrained Inertial Stabilizers
         5531,  # Beta Hull Mod Inertial Stabilizers
         5529,  # Local Hull Conversion Inertial Stabilizers I
         5527,  # Mark I Modified SS Inertial Stabilizers
         5519,  # Marked Modified SS Inertial Stabilizers
     ),
 
-    5239: (  # EP-S Gaussian Scoped Mining Laser
+    5239 : (  # EP-S Gaussian Scoped Mining Laser
         5241,  # Dual Diode Mining Laser I
     ),
-    5233: (  # Single Diode Basic Mining Laser
+    5233 : (  # Single Diode Basic Mining Laser
         5231,  # EP-R Argon Ion Basic Excavation Pulse
         5237,  # Rubin Basic Particle Bore Stream
         5235,  # Xenon Basic Drilling Beam
     ),
-    5245: (  # Particle Bore Compact Mining Laser
+    5245 : (  # Particle Bore Compact Mining Laser
         5243,  # XeCl Drilling Beam I
     ),
 
@@ -80,62 +79,63 @@ CONVERSIONS = {
         22609,  # Erin Mining Laser Upgrade
     ),
 
-    1242: (  # 'Basic' Nanofiber Internal Structure
+    1242 : (  # 'Basic' Nanofiber Internal Structure
         5591,  # Alpha Hull Mod Nanofiber Structure
         5595,  # Marked Modified SS Nanofiber Structure
         5559,  # Partial Hull Conversion Nanofiber Structure
         5593,  # Type-E Altered SS Nanofiber Structure
     ),
-    5599: (  # Type-D Restrained Nanofiber Structure
+    5599 : (  # Type-D Restrained Nanofiber Structure
         5597,  # Beta Hull Mod Nanofiber Structure
         5561,  # Local Hull Conversion Nanofiber Structure I
         5601,  # Mark I Modified SS Nanofiber Structure
     ),
 
-    1192: (  # 'Basic' Overdrive Injector System
+    1192 : (  # 'Basic' Overdrive Injector System
         5613,  # Alpha Hull Mod Overdrive Injector
         5617,  # Marked Modified SS Overdrive Injector
         5611,  # Partial Hull Conversion Overdrive Injector
         5615,  # Type-E Altered SS Overdrive Injector
     ),
-    5631: (  # Type-D Restrained Overdrive Injector
+    5631 : (  # Type-D Restrained Overdrive Injector
         5629,  # Beta Hull Mod Overdrive Injector
         5627,  # Local Hull Conversion Overdrive Injector I
         5633,  # Mark I Modified SS Overdrive Injector
     ),
 
-    1537: (  # 'Basic' Power Diagnostic System
+    1537 : (  # 'Basic' Power Diagnostic System
         8213,  # Alpha Reactor Control: Diagnostic System
         8217,  # Marked Generator Refitting: Diagnostic System
         8211,  # Partial Power Plant Manager: Diagnostic System
         8215,  # Type-E Power Core Modification: Diagnostic System
         8255,  # Type-E Power Core Modification: Reaction Control
     ),
-    8225: (  # Mark I Compact Power Diagnostic System
+    8225 : (  # Mark I Compact Power Diagnostic System
         8221,  # Beta Reactor Control: Diagnostic System I
         8219,  # Local Power Plant Manager: Diagnostic System I
         8223,  # Type-D Power Core Modification: Diagnostic System
     ),
 
-    1240: (  # 'Basic' Reinforced Bulkheads
+    1240 : (  # 'Basic' Reinforced Bulkheads
         5677,  # Alpha Hull Mod Reinforced Bulkheads
         5681,  # Marked Modified SS Reinforced Bulkheads
         5675,  # Partial Hull Conversion Reinforced Bulkheads
         5679,  # Type-E Altered SS Reinforced Bulkheads
     ),
-    5649: (  # Mark I Compact Reinforced Bulkheads
+    5649 : (  # Mark I Compact Reinforced Bulkheads
         5645,  # Beta Hull Mod Reinforced Bulkheads
     ),
-    5647: (  # Type-D Restrained Reinforced Bulkheads
+    5647 : (  # Type-D Restrained Reinforced Bulkheads
         5643,  # Local Hull Conversion Reinforced Bulkheads I
     ),
 }
 
+
 def upgrade(saveddata_engine):
-
     # Convert modules
-    for replacement_item, list in CONVERSIONS.iteritems():
+    for replacement_item, list in CONVERSIONS.items():
         for retired_item in list:
-            saveddata_engine.execute('UPDATE "modules" SET "itemID" = ? WHERE "itemID" = ?', (replacement_item, retired_item))
-            saveddata_engine.execute('UPDATE "cargo" SET "itemID" = ? WHERE "itemID" = ?', (replacement_item, retired_item))
-
+            saveddata_engine.execute('UPDATE "modules" SET "itemID" = ? WHERE "itemID" = ?',
+                                     (replacement_item, retired_item))
+            saveddata_engine.execute('UPDATE "cargo" SET "itemID" = ? WHERE "itemID" = ?',
+                                     (replacement_item, retired_item))

eos/db/migrations/upgrade40.py

@@ -0,0 +1,18 @@
+"""
+Migration 40
+
+Imports all item conversions since Migration 28 and runs them against module.baseItemID. This column seems to have been
+forgotten about since it's been added.
+
+"""
+from .upgrade36 import CONVERSIONS as u36
+from .upgrade37 import CONVERSIONS as u37
+from .upgrade38 import CONVERSIONS as u38
+from .upgrade39 import CONVERSIONS as u39
+
+def upgrade(saveddata_engine):
+    for conversions in [u36, u37, u38, u39]:
+        for replacement_item, list in conversions.items():
+            for retired_item in list:
+                saveddata_engine.execute('UPDATE "modules" SET "baseItemID" = ? WHERE "baseItemID" = ?',
+                               (replacement_item, retired_item))

eos/db/migrations/upgrade41.py

@@ -0,0 +1,50 @@
+"""
+Migration 41
+
+- Resistance plating tiericide
+"""
+
+CONVERSIONS = {
+    16345: (  # Upgraded Layered Coating I
+        16347,  # Limited Layered Plating I
+        16349,  # 'Scarab' Layered Plating I
+        16351,  # 'Grail' Layered Plating I
+    ),
+    16305: (  # Upgraded Multispectrum Coating I
+        16307,  # Limited Adaptive Nano Plating I
+        16309,  # 'Collateral' Adaptive Nano Plating I
+        16311,  # 'Refuge' Adaptive Nano Plating I
+    ),
+    16329: (  # Upgraded EM Coating I
+        16331,  # Limited EM Plating I
+        16333,  # 'Contour' EM Plating I
+        16335,  # 'Spiegel' EM Plating I
+    ),
+    16321: (  # Upgraded Explosive Coating I
+        16323,  # Limited Explosive Plating I
+        16325,  # Experimental Explosive Plating I
+        16319,  # 'Aegis' Explosive Plating I
+    ),
+    16313: (  # Upgraded Kinetic Coating I
+        16315,  # Limited Kinetic Plating I
+        16317,  # Experimental Kinetic Plating I
+        16327,  # 'Element' Kinetic Plating I
+    ),
+    16337: (  # Upgraded Thermal Coating I
+        16339,  # Limited Thermal Plating I
+        16341,  # Experimental Thermal Plating I
+        16343,  # Prototype Thermal Plating I
+    )
+}
+
+
+def upgrade(saveddata_engine):
+    # Convert modules
+    for replacement_item, list in CONVERSIONS.items():
+        for retired_item in list:
+            saveddata_engine.execute('UPDATE "modules" SET "itemID" = ? WHERE "itemID" = ?',
+                                     (replacement_item, retired_item))
+            saveddata_engine.execute('UPDATE "modules" SET "baseItemID" = ? WHERE "baseItemID" = ?',
+                                     (replacement_item, retired_item))
+            saveddata_engine.execute('UPDATE "cargo" SET "itemID" = ? WHERE "itemID" = ?',
+                                     (replacement_item, retired_item))

eos/db/migrations/upgrade42.py

@@ -0,0 +1,50 @@
+"""
+Migration 42
+
+- Resistance membrane tiericide
+"""
+
+CONVERSIONS = {
+    16391: (  # Compact Multispectrum Energized Membrane
+        16389,  # Experimental Energized Adaptive Nano Membrane I
+        16387,  # Limited Energized Adaptive Nano Membrane I
+        16385,  # Upgraded Energized Adaptive Nano Membrane I
+    ),
+    16423: (  # Compact Layered Energized Membrane
+        16421,  # Experimental Energized Armor Layering Membrane I
+        16419,  # Limited Energized Armor Layering Membrane I
+        16417,  # Upgraded Energized Armor Layering Membrane I
+    ),
+    16415: (  # Compact EM Energized Membrane
+        16413,  # Experimental Energized EM Membrane I
+        16411,  # Limited Energized EM Membrane I
+        16409,  # Upgraded Energized EM Membrane I
+    ),
+    16407: (  # Compact Explosive Energized Membrane
+        16405,  # Experimental Energized Explosive Membrane I
+        16403,  # Limited Energized Explosive Membrane I
+        16401,  # Upgraded Energized Explosive Membrane I
+    ),
+    16399: (  # Compact Kinetic Energized Membrane
+        16397,  # Experimental Energized Kinetic Membrane I
+        16395,  # Limited Energized Kinetic Membrane I
+        16393,  # Upgraded Energized Kinetic Membrane I
+    ),
+    16431: (  # Compact Thermal Energized Membrane
+        16429,  # Experimental Energized Thermal Membrane I
+        16427,  # Limited Energized Thermal Membrane I
+        16425,  # Upgraded Energized Thermal Membrane I
+    )
+}
+
+
+def upgrade(saveddata_engine):
+    # Convert modules
+    for replacement_item, list in CONVERSIONS.items():
+        for retired_item in list:
+            saveddata_engine.execute('UPDATE "modules" SET "itemID" = ? WHERE "itemID" = ?',
+                                     (replacement_item, retired_item))
+            saveddata_engine.execute('UPDATE "modules" SET "baseItemID" = ? WHERE "baseItemID" = ?',
+                                     (replacement_item, retired_item))
+            saveddata_engine.execute('UPDATE "cargo" SET "itemID" = ? WHERE "itemID" = ?',
+                                     (replacement_item, retired_item))

eos/db/migrations/upgrade43.py

@@ -0,0 +1,26 @@
+"""
+Migration 43
+
+- Shield booster amplifier tiericide
+"""
+
+CONVERSIONS = {
+    16533: (  # Stalwart Restrained Shield Boost Amplifier
+        16531,  # 5a Prototype Shield Support I
+    ),
+    16535: (  # Copasetic Compact Shield Boost Amplifier
+        16529,  # Ionic Field Accelerator I
+    ),
+}
+
+
+def upgrade(saveddata_engine):
+    # Convert modules
+    for replacement_item, list in CONVERSIONS.items():
+        for retired_item in list:
+            saveddata_engine.execute('UPDATE "modules" SET "itemID" = ? WHERE "itemID" = ?',
+                                     (replacement_item, retired_item))
+            saveddata_engine.execute('UPDATE "modules" SET "baseItemID" = ? WHERE "baseItemID" = ?',
+                                     (replacement_item, retired_item))
+            saveddata_engine.execute('UPDATE "cargo" SET "itemID" = ? WHERE "itemID" = ?',
+                                     (replacement_item, retired_item))

eos/db/migrations/upgrade44.py

@@ -0,0 +1,25 @@
+"""
+Migration 44
+
+- Signal distortion amplifier tiericide
+"""
+
+CONVERSIONS = {
+    25565: (  # Hypnos Compact Signal Distortion Amplifier I
+        25571,  # Initiated Signal Distortion Amplifier I
+        25569,  # Induced Signal Distortion Amplifier I
+        25567,  # Compulsive Signal Distortion Amplifier I
+    ),
+}
+
+
+def upgrade(saveddata_engine):
+    # Convert modules
+    for replacement_item, list in CONVERSIONS.items():
+        for retired_item in list:
+            saveddata_engine.execute('UPDATE "modules" SET "itemID" = ? WHERE "itemID" = ?',
+                                     (replacement_item, retired_item))
+            saveddata_engine.execute('UPDATE "modules" SET "baseItemID" = ? WHERE "baseItemID" = ?',
+                                     (replacement_item, retired_item))
+            saveddata_engine.execute('UPDATE "cargo" SET "itemID" = ? WHERE "itemID" = ?',
+                                     (replacement_item, retired_item))

eos/db/migrations/upgrade45.py

@@ -0,0 +1,18 @@
+"""
+Migration 45
+
+- Drone mutaplasmid support
+"""
+
+import sqlalchemy
+
+
+def upgrade(saveddata_engine):
+    try:
+        saveddata_engine.execute("SELECT baseItemID FROM drones LIMIT 1")
+    except sqlalchemy.exc.DatabaseError:
+        saveddata_engine.execute("ALTER TABLE drones ADD COLUMN baseItemID INTEGER;")
+    try:
+        saveddata_engine.execute("SELECT mutaplasmidID FROM drones LIMIT 1")
+    except sqlalchemy.exc.DatabaseError:
+        saveddata_engine.execute("ALTER TABLE drones ADD COLUMN mutaplasmidID INTEGER;")

eos/db/migrations/upgrade46.py

@@ -0,0 +1,68 @@
+"""
+Migration 46
+
+- Mining crystal changes
+"""
+
+CONVERSIONS = {
+    60276: (  # Simple Asteroid Mining Crystal Type A I
+        18066,  # Veldspar Mining Crystal I
+        18062,  # Scordite Mining Crystal I
+        18060,  # Pyroxeres Mining Crystal I
+        18058,  # Plagioclase Mining Crystal I
+    ),
+    60281: (  # Simple Asteroid Mining Crystal Type A II
+        18618,  # Veldspar Mining Crystal II
+        18616,  # Scordite Mining Crystal II
+        18614,  # Pyroxeres Mining Crystal II
+        18612,  # Plagioclase Mining Crystal II
+    ),
+    60285: (  # Coherent Asteroid Mining Crystal Type A I
+        18056,  # Omber Mining Crystal I
+        18052,  # Kernite Mining Crystal I
+        18050,  # Jaspet Mining Crystal I
+        18048,  # Hemorphite Mining Crystal I
+        18046,  # Hedbergite Mining Crystal I
+    ),
+    60288: (  # Coherent Asteroid Mining Crystal Type A II
+        18610,  # Omber Mining Crystal II
+        18604,  # Jaspet Mining Crystal II
+        18606,  # Kernite Mining Crystal II
+        18600,  # Hedbergite Mining Crystal II
+        18602,  # Hemorphite Mining Crystal II
+    ),
+    60291: (  # Variegated Asteroid Mining Crystal Type A I
+        18044,  # Gneiss Mining Crystal I
+        18042,  # Dark Ochre Mining Crystal I
+        18040,  # Crokite Mining Crystal I
+    ),
+    60294: (  # Variegated Asteroid Mining Crystal Type A II
+        18598,  # Gneiss Mining Crystal II
+        18596,  # Dark Ochre Mining Crystal II
+        18594,  # Crokite Mining Crystal II
+    ),
+    60297: (  # Complex Asteroid Mining Crystal Type A I
+        18038,  # Bistot Mining Crystal I
+        18036,  # Arkonor Mining Crystal I
+        18064,  # Spodumain Mining Crystal I
+    ),
+    60300: (  # Complex Asteroid Mining Crystal Type A II
+        18592,  # Bistot Mining Crystal II
+        18590,  # Arkonor Mining Crystal II
+        18624,  # Spodumain Mining Crystal II
+    ),
+}
+
+
+def upgrade(saveddata_engine):
+    # Convert modules
+    for replacement_item, list in CONVERSIONS.items():
+        for retired_item in list:
+            saveddata_engine.execute('UPDATE "modules" SET "itemID" = ? WHERE "itemID" = ?',
+                                     (replacement_item, retired_item))
+            saveddata_engine.execute('UPDATE "modules" SET "baseItemID" = ? WHERE "baseItemID" = ?',
+                                     (replacement_item, retired_item))
+            saveddata_engine.execute('UPDATE "modules" SET "chargeID" = ? WHERE "chargeID" = ?',
+                                     (replacement_item, retired_item))
+            saveddata_engine.execute('UPDATE "cargo" SET "itemID" = ? WHERE "itemID" = ?',
+                                     (replacement_item, retired_item))

eos/db/migrations/upgrade47.py

@@ -0,0 +1,36 @@
+"""
+Migration 46
+
+- add support for server selection for SSO characters
+"""
+import sqlalchemy
+
+tmpTable = """
+    CREATE TABLE ssoCharacterTemp (
+        ID INTEGER NOT NULL, 
+        client VARCHAR NOT NULL, 
+        characterID INTEGER NOT NULL, 
+        characterName VARCHAR NOT NULL, 
+        refreshToken VARCHAR NOT NULL, 
+        accessToken VARCHAR NOT NULL, 
+        accessTokenExpires DATETIME NOT NULL, 
+        created DATETIME, 
+        modified DATETIME,
+        server VARCHAR, 
+        PRIMARY KEY (ID), 
+        CONSTRAINT "uix_client_server_characterID" UNIQUE (client, server, characterID), 
+        CONSTRAINT "uix_client_server_characterName" UNIQUE (client, server, characterName)
+    )
+"""
+
+def upgrade(saveddata_engine):
+    try:
+        saveddata_engine.execute("SELECT server FROM ssoCharacter LIMIT 1")
+    except sqlalchemy.exc.DatabaseError:
+        saveddata_engine.execute(tmpTable)
+        saveddata_engine.execute(
+            "INSERT INTO ssoCharacterTemp (ID, client, characterID, characterName, refreshToken, accessToken, accessTokenExpires, created, modified, server) "
+            "SELECT ID, client, characterID, characterName, refreshToken, accessToken, accessTokenExpires, created, modified, 'Tranquility' "
+            "FROM ssoCharacter")
+        saveddata_engine.execute("DROP TABLE ssoCharacter")
+        saveddata_engine.execute("ALTER TABLE ssoCharacterTemp RENAME TO ssoCharacter")

eos/db/migrations/upgrade48.py

@@ -0,0 +1,15 @@
+"""
+Migration 48
+
+- added pilot security column (CONCORD ships)
+"""
+
+
+import sqlalchemy
+
+
+def upgrade(saveddata_engine):
+    try:
+        saveddata_engine.execute("SELECT pilotSecurity FROM fits LIMIT 1")
+    except sqlalchemy.exc.DatabaseError:
+        saveddata_engine.execute("ALTER TABLE fits ADD COLUMN pilotSecurity FLOAT")

eos/db/migrations/upgrade49.py

@@ -0,0 +1,15 @@
+"""
+Migration 49
+
+- added hp column to targetResists table
+"""
+
+
+import sqlalchemy
+
+
+def upgrade(saveddata_engine):
+        try:
+            saveddata_engine.execute("SELECT hp FROM targetResists LIMIT 1;")
+        except sqlalchemy.exc.DatabaseError:
+            saveddata_engine.execute("ALTER TABLE targetResists ADD COLUMN hp FLOAT;")

eos/db/migrations/upgrade5.py

@@ -4,5 +4,6 @@ Migration 5
 Simply deletes damage profiles with a blank name. See GH issue #256
 """
 
+
 def upgrade(saveddata_engine):
     saveddata_engine.execute('DELETE FROM damagePatterns WHERE name LIKE ?', ("",))

eos/db/migrations/upgrade6.py

@@ -4,6 +4,8 @@ Migration 6
 Overwrites damage profile 0 to reset bad uniform values (bad values set with bug)
 """
 
+
 def upgrade(saveddata_engine):
     saveddata_engine.execute('DELETE FROM damagePatterns WHERE name LIKE ? OR ID LIKE ?', ("Uniform", "1"))
-    saveddata_engine.execute('INSERT INTO damagePatterns VALUES (?, ?, ?, ?, ?, ?, ?)', (1, "Uniform", 25, 25, 25, 25, None))
+    saveddata_engine.execute('INSERT INTO damagePatterns (ID, name, emAmount, thermalAmount, kineticAmount, explosiveAmount, ownerID) VALUES (?, ?, ?, ?, ?, ?, ?)',
+                             (1, "Uniform", 25, 25, 25, 25, None))

eos/db/migrations/upgrade7.py

@@ -8,17 +8,16 @@ Migration 7
     Pyfa.
 """
 
-
 CONVERSIONS = {
     640: (  # Scorpion
         4005,  # Scorpion Ishukone Watch
     )
 }
 
+
 def upgrade(saveddata_engine):
-
     # Convert ships
-    for replacement_item, list in CONVERSIONS.iteritems():
+    for replacement_item, list in CONVERSIONS.items():
         for retired_item in list:
-            saveddata_engine.execute('UPDATE "fits" SET "shipID" = ? WHERE "shipID" = ?', (replacement_item, retired_item))
-
+            saveddata_engine.execute('UPDATE "fits" SET "shipID" = ? WHERE "shipID" = ?',
+                                     (replacement_item, retired_item))