diff --git a/scripts/compile_data.py b/scripts/compile_data.py new file mode 100644 index 000000000..9de7c5562 --- /dev/null +++ b/scripts/compile_data.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python +""" +This script bootstraps Phobos from a supplied path and feeds it +information regarding EVE data paths and where to dump data. It then imports +some other scripts and uses them to convert the json data into a SQLite +database and then compare the new database to the existing one, producing a +diff which can then be used to assist in the updating. +""" + +import sys +import os + +import argparse + +parser = argparse.ArgumentParser() +parser.add_argument("-d", "--dump", dest="dump_path", help="Location of Phobos JSON dump directory", required=True) + +args = parser.parse_args() +dump_path = os.path.expanduser(args.dump_path) +script_path = os.path.dirname(__file__) + +def header(text, subtext=None): + print() + print("* "*30) + print(text.center(60)) + if subtext: + print(subtext.center(60)) + print("* "*30) + print() + +### SQL Convert +import jsonToSql + +db_file = os.path.join(dump_path, "eve.db") +header("Converting Data to SQL", db_file) + +if os.path.isfile(db_file): + os.remove(db_file) + +jsonToSql.main("sqlite:///" + db_file, dump_path) + +### Diff generation +import itemDiff + +diff_file = os.path.join(dump_path, "diff.txt") +old_db = os.path.join(script_path, "..", "eve.db") + +header("Generating DIFF", diff_file) +old_stdout = sys.stdout +sys.stdout = open(diff_file, 'w') +itemDiff.main(old=old_db, new=db_file) +sys.stdout = old_stdout + +header("Commiting changes for ", diff_file) + +from subprocess import call + +os.chdir(dump_path) + +call(["git.exe", "add", "."]) +call(["git.exe", "commit", "-m", "Commit"]) + +print("\nAll done.") \ No newline at end of file diff --git a/scripts/dump_data.py b/scripts/dump_data.py new file mode 100644 index 000000000..2393fb8c4 --- /dev/null +++ b/scripts/dump_data.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python +""" +This script bootstraps Phobos from a supplied path and feeds it +information regarding EVE data paths and where to dump data. It then imports +some other scripts and uses them to convert the json data into a SQLite +database and then compare the new database to the existing one, producing a +diff which can then be used to assist in the updating. +""" + +import sys +import os + +# Phobos location +phb_path = os.path.expanduser("path/to/phobos") + +import argparse + +parser = argparse.ArgumentParser() +parser.add_argument("-e", "--eve", dest="eve_path", help="Location of EVE directory", required=True) +parser.add_argument("-c", "--cache", dest="cache_path", help="Location of EVE cache directory. If not specified, an attempt will be make to automatically determine path.") +parser.add_argument("-r", "--res", dest="res_path", help="Location of EVE shared resource cache. If not specified, an attempt will be make to automatically determine path.") +parser.add_argument("-d", "--dump", dest="dump_path", help="Location of Phobos JSON dump directory", required=True) +parser.add_argument("-p", "--phobos", dest="phb_path", help="Location of Phobos, defaults to path noted in script", default=phb_path) +parser.add_argument("-s", "--singularity", action="store_true", help="Singularity build") + +args = parser.parse_args() +eve_path = os.path.expanduser(args.eve_path) +cache_path = os.path.expanduser(args.cache_path) if args.cache_path else None +res_path = os.path.expanduser(args.res_path) if args.res_path else None +dump_path = os.path.expanduser(args.dump_path) +script_path = os.path.dirname(__file__) + +### Append Phobos to path +sys.path.append(os.path.expanduser(args.phb_path)) + +def header(text, subtext=None): + print() + print("* "*30) + print(text.center(60)) + if subtext: + print(subtext.center(60)) + print("* "*30) + print() + +header("Dumping Phobos Data", dump_path) + +import reverence +from flow import FlowManager +from miner import * +from translator import Translator +from writer import * + +rvr = reverence.blue.EVE(eve_path, cachepath=args.cache_path, sharedcachepath=res_path, server="singularity" if args.singularity else "tranquility") +print("EVE Directory: {}".format(rvr.paths.root)) +print("Cache Directory: {}".format(rvr.paths.cache)) +print("Shared Resource Directory: {}".format(rvr.paths.sharedcache)) + +pickle_miner = ResourcePickleMiner(rvr) +trans = Translator(pickle_miner) +bulkdata_miner = BulkdataMiner(rvr, trans) +staticcache_miner = ResourceStaticCacheMiner(rvr, trans) +miners = ( + MetadataMiner(eve_path), + bulkdata_miner, + staticcache_miner, + TraitMiner(staticcache_miner, bulkdata_miner, trans), + SqliteMiner(rvr.paths.root, trans), + CachedCallsMiner(rvr, trans), + pickle_miner +) + +writers = ( + JsonWriter(dump_path, indent=2), +) + +list = "dgmexpressions,dgmattribs,dgmeffects,dgmtypeattribs,dgmtypeeffects,"\ + "dgmunits,invcategories,invgroups,invmetagroups,invmetatypes,"\ + "invtypes,mapbulk_marketGroups,phbmetadata,phbtraits,fsdTypeOverrides,"\ + "evegroups,evetypes,evecategories,mapbulk_marketGroups,clonegrades" + +FlowManager(miners, writers).run(list, "en-us") diff --git a/scripts/jsonToSql.py b/scripts/jsonToSql.py index 6629c62b2..717713e64 100755 --- a/scripts/jsonToSql.py +++ b/scripts/jsonToSql.py @@ -20,16 +20,19 @@ import os import sys +import functools import re # Add eos root path to sys.path so we can import ourselves -path = os.path.dirname(str(__file__, sys.getfilesystemencoding())) +path = os.path.dirname(__file__) sys.path.append(os.path.realpath(os.path.join(path, ".."))) import json import argparse def main(db, json_path): + if os.path.isfile(db): + os.remove(db) jsonPath = os.path.expanduser(json_path) @@ -130,7 +133,7 @@ def main(db, json_path): check[ID] = {} check[ID][int(skill["typeID"])] = int(skill["level"]) - if not reduce(lambda a, b: a if a == b else False, [v for _, v in check.iteritems()]): + if not functools.reduce(lambda a, b: a if a == b else False, [v for _, v in check.items()]): raise Exception("Alpha Clones not all equal") newData = [x for x in newData if x['alphaCloneID'] == 1] @@ -188,7 +191,7 @@ def main(db, json_path): # Dump all data to memory so we can easely cross check ignored rows for jsonName, cls in tables.items(): - with open(os.path.join(jsonPath, "{}.json".format(jsonName))) as f: + with open(os.path.join(jsonPath, "{}.json".format(jsonName)), encoding="utf-8") as f: tableData = json.load(f) if jsonName in rowsInValues: tableData = list(tableData.values()) diff --git a/scripts/prep_data.py b/scripts/prep_data.py deleted file mode 100644 index d8907aa58..000000000 --- a/scripts/prep_data.py +++ /dev/null @@ -1,109 +0,0 @@ -#!/usr/bin/env python -""" -This script bootstraps Phobos from a supplied path and feeds it -information regarding EVE data paths and where to dump data. It then imports -some other scripts and uses them to convert the json data into a SQLite -database and then compare the new database to the existing one, producing a -diff which can then be used to assist in the updating. -""" - -import sys -import os - -# Phobos location -phb_path = os.path.expanduser("path/to/phobos") - -import argparse - -parser = argparse.ArgumentParser() -parser.add_argument("-e", "--eve", dest="eve_path", help="Location of EVE directory", required=True) -parser.add_argument("-c", "--cache", dest="cache_path", help="Location of EVE cache directory. If not specified, an attempt will be make to automatically determine path.") -parser.add_argument("-r", "--res", dest="res_path", help="Location of EVE shared resource cache. If not specified, an attempt will be make to automatically determine path.") -parser.add_argument("-d", "--dump", dest="dump_path", help="Location of Phobos JSON dump directory", required=True) -parser.add_argument("-p", "--phobos", dest="phb_path", help="Location of Phobos, defaults to path noted in script", default=phb_path) -parser.add_argument("-s", "--singularity", action="store_true", help="Singularity build") -parser.add_argument("-j", "--nojson", dest="nojson", action="store_true", help="Skip Phobos JSON data dump.") - -args = parser.parse_args() -eve_path = os.path.expanduser(str(args.eve_path, sys.getfilesystemencoding())) -cache_path = os.path.expanduser(str(args.cache_path, sys.getfilesystemencoding())) if args.cache_path else None -res_path = os.path.expanduser(str(args.res_path, sys.getfilesystemencoding())) if args.res_path else None -dump_path = os.path.expanduser(str(args.dump_path, sys.getfilesystemencoding())) -script_path = os.path.dirname(str(__file__, sys.getfilesystemencoding())) - -### Append Phobos to path -sys.path.append(os.path.expanduser(str(args.phb_path, sys.getfilesystemencoding()))) - -def header(text, subtext=None): - print() - print("* "*30) - print(text.center(60)) - if subtext: - print(subtext.center(60)) - print("* "*30) - print() - -### Data dump -if not args.nojson: - header("Dumping Phobos Data", dump_path) - - import reverence - from flow import FlowManager - from miner import * - from translator import Translator - from writer import * - - rvr = reverence.blue.EVE(eve_path, cachepath=args.cache_path, sharedcachepath=res_path, server="singularity" if args.singularity else "tranquility") - print("EVE Directory: {}".format(rvr.paths.root)) - print("Cache Directory: {}".format(rvr.paths.cache)) - print("Shared Resource Directory: {}".format(rvr.paths.sharedcache)) - print() - - pickle_miner = ResourcePickleMiner(rvr) - trans = Translator(pickle_miner) - bulkdata_miner = BulkdataMiner(rvr, trans) - staticcache_miner = ResourceStaticCacheMiner(rvr, trans) - miners = ( - MetadataMiner(eve_path), - bulkdata_miner, - staticcache_miner, - TraitMiner(staticcache_miner, bulkdata_miner, trans), - SqliteMiner(rvr.paths.root, trans), - CachedCallsMiner(rvr, trans), - pickle_miner - ) - - writers = ( - JsonWriter(dump_path, indent=2), - ) - - list = "dgmexpressions,dgmattribs,dgmeffects,dgmtypeattribs,dgmtypeeffects,"\ - "dgmunits,invcategories,invgroups,invmetagroups,invmetatypes,"\ - "invtypes,mapbulk_marketGroups,phbmetadata,phbtraits,fsdTypeOverrides,"\ - "evegroups,evetypes,evecategories,mapbulk_marketGroups,clonegrades" - - FlowManager(miners, writers).run(list, "en-us") - -### SQL Convert -import jsonToSql - -db_file = os.path.join(dump_path, "eve.db") -header("Converting Data to SQL", db_file) - -if os.path.isfile(db_file): - os.remove(db_file) - -jsonToSql.main("sqlite:///"+db_file, dump_path) - -### Diff generation -import itemDiff -diff_file = os.path.join(dump_path, "diff.txt") -old_db = os.path.join(script_path, "..", "eve.db") - -header("Generating DIFF", diff_file) -old_stdout = sys.stdout -sys.stdout = open(diff_file, 'w') -itemDiff.main(old=old_db, new=db_file) -sys.stdout = old_stdout - -print("\nAll done.")