Added script to automate json dump, sql convert, and diff prep. Also adds metadata to eve.db

2014-11-29 20:50:34 -05:00
parent b93b59fe4d
commit a565abe6f7
6 changed files with 561 additions and 447 deletions
--- a/scripts/prep_data.py
+++ b/scripts/prep_data.py
@@ -0,0 +1,93 @@
+#!/usr/bin/env python
+
+import sys
+import os
+
+# Phobos location
+phb_path = os.path.expanduser("path/to/phobos")
+
+### Append Phobos to path
+sys.path.append(os.path.realpath(phb_path))
+
+import argparse
+
+parser = argparse.ArgumentParser()
+parser.add_argument("-e", "--eve", dest="eve_path", help="Location of EVE directory", required=True)
+parser.add_argument("-c", "--cache", dest="cache_path", help="Location of EVE cache directory. If not specified, an attempt will be make to automatically determine path.")
+parser.add_argument("-d", "--dump", dest="dump_path", help="Location of Phobos JSON dump directory", required=True)
+parser.add_argument("-s", "--singularity", action="store_true", help="Singularity build")
+parser.add_argument("-j", "--nojson", dest="nojson", action="store_true", help="Skip Phobos JSON data dump.")
+
+args = parser.parse_args()
+eve_path = os.path.expanduser(unicode(args.eve_path, sys.getfilesystemencoding()))
+cache_path = os.path.expanduser(unicode(args.cache_path, sys.getfilesystemencoding())) if args.cache_path else None
+dump_path = os.path.expanduser(unicode(args.dump_path, sys.getfilesystemencoding()))
+script_path = os.path.dirname(unicode(__file__, sys.getfilesystemencoding()))
+
+def header(text, subtext=None):
+    print
+    print "* "*30
+    print text.center(60)
+    if subtext:
+        print subtext.center(60)
+    print "* "*30
+    print
+
+### Data dump
+if not args.nojson:
+    header("Dumping Phobos Data", dump_path)
+
+    import reverence
+    from flow import FlowManager
+    from miner import *
+    from translator import Translator
+    from writer import *
+
+    rvr = reverence.blue.EVE(eve_path, cachepath=args.cache_path, server="singularity" if args.singularity else "tranquility")
+
+    spickle_miner = StuffedPickleMiner(rvr)
+    trans = Translator(spickle_miner)
+    bulkdata_miner = BulkdataMiner(rvr, trans)
+
+    miners = (
+        MetadataMiner(eve_path),
+        bulkdata_miner,
+        TraitMiner(bulkdata_miner, trans),
+        SqliteMiner(eve_path, trans),
+        CachedCallsMiner(rvr, trans),
+        spickle_miner
+    )
+
+    writers = (
+        JsonWriter(dump_path, indent=2),
+    )
+
+    list = "dgmexpressions,dgmattribs,dgmeffects,dgmtypeattribs,dgmtypeeffects,"\
+           "dgmunits,icons,invcategories,invgroups,invmetagroups,invmetatypes,"\
+           "invtypes,mapbulk_marketGroups,phbmetadata,phbtraits"
+
+    FlowManager(miners, writers).run(list, "multi")
+
+### SQL Convert
+import jsonToSql
+
+db_file = os.path.join(dump_path, "eve.db")
+header("Converting Data to SQL", db_file)
+
+if os.path.isfile(db_file):
+    os.remove(db_file)
+
+jsonToSql.main("sqlite:///"+db_file, dump_path)
+
+### Diff generation
+import itemDiff
+diff_file = os.path.join(dump_path, "diff.txt")
+old_db = os.path.join(script_path, "..", "pyfa", "staticdata", "eve.db")
+
+header("Generating DIFF", diff_file)
+old_stdout = sys.stdout
+sys.stdout = open(diff_file, 'w')
+itemDiff.main(old=old_db, new=db_file)
+sys.stdout = old_stdout
+
+print "\nAll done."