Replace submodules with actual files
Submodules never were actually useful
This commit is contained in:
DarkPhoenix
350
eos/utils/scripts/eveCacheToDb.py
Executable file
350
eos/utils/scripts/eveCacheToDb.py
Executable file
@@ -0,0 +1,350 @@
|
||||
#!/usr/bin/env python
|
||||
#===============================================================================
|
||||
# Copyright (C) 2010 Anton Vorobyov
|
||||
# Copyright (C) 2010 Diego Duclos
|
||||
#
|
||||
# This file is part of eos.
|
||||
#
|
||||
# eos is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License as published by
|
||||
# the Free Software Foundation, either version 2 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# eos is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public License
|
||||
# along with eos. If not, see <http://www.gnu.org/licenses/>.
|
||||
#===============================================================================
|
||||
|
||||
'''
|
||||
This script pulls data out of EVE cache and makes a database dump. To get most of the data,
|
||||
you need to just log into game; however, for some special data sometimes you need to dump
|
||||
it by executing corresponding action in game, for example - open market tree to get data for
|
||||
invmarketgroups table.
|
||||
Reverence library by Entity is used, check http://wiki.github.com/ntt/reverence/ for info
|
||||
As reverence uses the same Python version as EVE client (2.x series), script cannot be converted to python3
|
||||
Example commands to run the script under Linux with default eve paths for getting SQLite dump:
|
||||
Tranquility: python eveCacheToDb.py --eve="~/.wine/drive_c/Program Files/CCP/EVE" --cache="~/.wine/drive_c/users/"$USER"/Local Settings/Application Data/CCP/EVE/c_program_files_ccp_eve_tranquility/cache" --dump="sqlite:////home/"$USER"/Desktop/eve.db"
|
||||
Singularity: python eveCacheToDb.py --eve="~/.wine/drive_c/Program Files/CCP/Singularity" --cache="~/.wine/drive_c/users/"$USER"/Local Settings/Application Data/CCP/EVE/c_program_files_ccp_singularity_singularity/cache" --sisi --dump="sqlite:////home/"$USER"/Desktop/evetest.db"
|
||||
'''
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
# Add eos root path to sys.path
|
||||
path = os.path.dirname(unicode(__file__, sys.getfilesystemencoding()))
|
||||
sys.path.append(os.path.realpath(os.path.join(path, "..", "..", "..")))
|
||||
|
||||
def get_map():
|
||||
"""
|
||||
Return table name - table class map
|
||||
"""
|
||||
return {"allianceshortnames": None,
|
||||
"billtypes": None,
|
||||
"certificaterelationships": None,
|
||||
"certificates": None,
|
||||
"corptickernames": None,
|
||||
"dgmattribs": AttributeInfo,
|
||||
"dgmeffects": EffectInfo,
|
||||
"dgmtypeattribs": Attribute,
|
||||
"dgmtypeeffects": Effect,
|
||||
"evegraphics": None,
|
||||
"evelocations": None,
|
||||
"eveowners": None,
|
||||
"eveunits": Unit,
|
||||
"groupsByCategories": None,
|
||||
"icons": Icon,
|
||||
"invbptypes": None,
|
||||
"invcategories": Category,
|
||||
"invcontrabandTypesByFaction": None,
|
||||
"invcontrabandTypesByType": None,
|
||||
"invgroups": Group,
|
||||
"invmetagroups": MetaGroup,
|
||||
"invmarketgroups": MarketGroup,
|
||||
"invmetatypes": MetaType,
|
||||
"invmetatypesByTypeID": None,
|
||||
"invreactiontypes": None,
|
||||
"invtypes": Item,
|
||||
"locationscenes": None,
|
||||
"locationwormholeclasses": None,
|
||||
"mapcelestialdescriptions": None,
|
||||
"ownericons": None,
|
||||
"ramactivities": None,
|
||||
"ramaltypes": None,
|
||||
"ramaltypesdetailpercategory": None,
|
||||
"ramaltypesdetailpergroup": None,
|
||||
"ramcompletedstatuses": None,
|
||||
"ramtyperequirements": None,
|
||||
"schematics": None,
|
||||
"schematicsByPin": None,
|
||||
"schematicsByType": None,
|
||||
"schematicspinmap": None,
|
||||
"schematicstypemap": None,
|
||||
"shiptypes": None,
|
||||
"sounds": None,
|
||||
"typesByGroups": None,
|
||||
"typesByMarketGroups": None}
|
||||
|
||||
def get_order():
|
||||
"""
|
||||
Return order for table processing
|
||||
"""
|
||||
return ("icons",
|
||||
"invmarketgroups",
|
||||
"eveunits",
|
||||
"dgmattribs",
|
||||
"dgmeffects",
|
||||
"invcategories",
|
||||
"invgroups",
|
||||
"invmetagroups",
|
||||
"invtypes",
|
||||
"invmetatypes",
|
||||
"dgmtypeattribs",
|
||||
"dgmtypeeffects")
|
||||
|
||||
def get_customcalls():
|
||||
"""
|
||||
Return custom table - call to get data for it map
|
||||
"""
|
||||
return {"invmarketgroups": eve.RemoteSvc("marketProxy").GetMarketGroups()}
|
||||
|
||||
def process_table(sourcetable, tablename, tableclass):
|
||||
"""
|
||||
Get all data from cache and write it to database
|
||||
"""
|
||||
# Get data from source and process it
|
||||
tabledata = get_table_data(sourcetable, tablename, get_source_headers(sourcetable))
|
||||
# Insert everything into table
|
||||
insert_table_values(tabledata, tableclass)
|
||||
return
|
||||
|
||||
def get_source_headers(sourcetable):
|
||||
"""
|
||||
Pull list of headers from the source table
|
||||
"""
|
||||
sourceheaders = None
|
||||
guid = getattr(sourcetable, "__guid__", "None")
|
||||
# For IndexRowset and IndexedRowLists Reverence provides list of headers
|
||||
if guid in ("util.IndexRowset", "util.FilterRowset"):
|
||||
sourceheaders = tuple(sourcetable.header)
|
||||
# For IndexedRowLists, we need to compose list ourselves
|
||||
elif guid == "util.IndexedRowLists":
|
||||
headerset = set()
|
||||
for item in sourcetable:
|
||||
for row in sourcetable[item]:
|
||||
for headername in row.__header__.Keys():
|
||||
headerset.add(headername)
|
||||
sourceheaders = tuple(headerset)
|
||||
return sourceheaders
|
||||
|
||||
def get_table_data(sourcetable, tablename, headers):
|
||||
"""
|
||||
Pull data out of source table
|
||||
"""
|
||||
# Each row is enclosed into dictionary, full table is list of these dictionaries
|
||||
datarows = []
|
||||
guid = getattr(sourcetable, "__guid__", "None")
|
||||
# We have Select method for IndexRowset tables
|
||||
if guid == "util.IndexRowset":
|
||||
for values in sourcetable.Select(*headers):
|
||||
# When Select is asked to find single value, it is returned in its raw
|
||||
# form. Convert is to tuple for proper further processing
|
||||
if not isinstance(values, (list, tuple, set)):
|
||||
values = (values,)
|
||||
headerslen = len(headers)
|
||||
datarow = {}
|
||||
# 1 row value should correspond to 1 header, if number or values doesn't
|
||||
# correspond to number of headers then something went wrong
|
||||
if headerslen != len(values):
|
||||
print "Error: malformed data in source table {0}".format(tablename)
|
||||
return None
|
||||
# Fill row dictionary with values and append it to list
|
||||
for i in xrange(headerslen):
|
||||
# If we've got ASCII string, convert it to Unicode
|
||||
if isinstance(values[i], str):
|
||||
datarow[headers[i]] = unicode(values[i], 'ISO-8859-1')
|
||||
else:
|
||||
datarow[headers[i]] = values[i]
|
||||
datarows.append(datarow)
|
||||
# FilterRowset and IndexedRowLists are accessible almost like dictionaries
|
||||
elif guid in ("util.FilterRowset", "util.IndexedRowLists"):
|
||||
# Go through all source table elements
|
||||
for element in sourcetable.iterkeys():
|
||||
# Go through all rows of an element
|
||||
for row in sourcetable[element]:
|
||||
datarow = {}
|
||||
# Fill row dictionary with values we need and append it to the list
|
||||
for header in headers:
|
||||
value = getattr(row, header, None)
|
||||
# None and zero values are different, and we want to write zero
|
||||
# values to database
|
||||
if value or value in (0, 0.0):
|
||||
datarow[header] = value
|
||||
datarows.append(datarow)
|
||||
|
||||
return datarows
|
||||
|
||||
def insert_table_values(tabledata, tableclass):
|
||||
"""
|
||||
Insert values into tables and show progress
|
||||
"""
|
||||
rows = 0
|
||||
rows_skipped = 0
|
||||
# Go through all table rows
|
||||
for row in tabledata:
|
||||
instance = tableclass()
|
||||
# Print dot each 1k inserted rows
|
||||
if rows / 1000.0 == int(rows / 1000.0):
|
||||
sys.stdout.write(".")
|
||||
sys.stdout.flush()
|
||||
try:
|
||||
# Go through all fields of a row, process them and insert
|
||||
for header in row:
|
||||
setattr(instance, header, process_value(row[header], tableclass, header))
|
||||
eos.db.gamedata_session.add(instance)
|
||||
rows += 1
|
||||
except ValueError:
|
||||
rows_skipped += 1
|
||||
# Print out results and actually commit results to database
|
||||
print "\nInserted {0} rows. skipped {1} rows".format(rows, rows_skipped)
|
||||
eos.db.gamedata_session.commit()
|
||||
|
||||
def process_value(value, tableclass, header):
|
||||
# Get column info
|
||||
info = tableclass._sa_class_manager.mapper.c.get(header)
|
||||
if info is None:
|
||||
return
|
||||
|
||||
# Null out non-existent foreign key relations
|
||||
foreign_keys = info.foreign_keys
|
||||
if len(foreign_keys) > 0:
|
||||
for key in foreign_keys:
|
||||
col = key.column
|
||||
if not query_existence(col, value) and not key.deferrable:
|
||||
if info.nullable:
|
||||
return None
|
||||
else:
|
||||
raise ValueError("Integrity check failed")
|
||||
else:
|
||||
return value
|
||||
#Turn booleans into actual booleans, don't leave them as integers
|
||||
elif type(info.type) == Boolean:
|
||||
return bool(value)
|
||||
else:
|
||||
return value
|
||||
|
||||
existence_cache = {}
|
||||
def query_existence(col, value):
|
||||
key = (col, col.table, value)
|
||||
info = existence_cache.get(key)
|
||||
if info is None:
|
||||
info = eos.db.gamedata_session.query(col.table).filter(col == value).count() > 0
|
||||
existence_cache[key] = info
|
||||
|
||||
return info
|
||||
|
||||
if __name__ == "__main__":
|
||||
from ConfigParser import ConfigParser
|
||||
from optparse import OptionParser
|
||||
|
||||
from reverence import blue
|
||||
from sqlalchemy import Boolean
|
||||
from sqlalchemy.orm import class_mapper, ColumnProperty
|
||||
|
||||
import eos.config
|
||||
|
||||
# Parse command line options
|
||||
usage = "usage: %prog --eve=EVE --cache=CACHE --dump=DUMP [--release=RELEASE --sisi]"
|
||||
parser = OptionParser(usage=usage)
|
||||
parser.add_option("-e", "--eve", help="path to eve folder")
|
||||
parser.add_option("-c", "--cache", help="path to eve cache folder")
|
||||
parser.add_option("-d", "--dump", help="the SQL Alchemy connection string of where we should place our final dump")
|
||||
parser.add_option("-r", "--release", help="database release number, defaults to 1", default="1")
|
||||
parser.add_option("-s", "--sisi", action="store_true", dest="singularity", help="if you're going to work with Singularity test server data, use this option", default=False)
|
||||
(options, args) = parser.parse_args()
|
||||
|
||||
|
||||
# Exit if we do not have any of required options
|
||||
if not options.eve or not options.cache or not options.dump:
|
||||
sys.stderr.write("You need to specify paths to eve folder, cache folder and SQL Alchemy connection string. Run script with --help option for further info.\n")
|
||||
sys.exit()
|
||||
|
||||
# We can deal either with singularity or tranquility servers
|
||||
if options.singularity: server = "singularity"
|
||||
else: server = "tranquility"
|
||||
|
||||
# Set static variables for paths
|
||||
PATH_EVE = os.path.expanduser(options.eve)
|
||||
PATH_CACHE = os.path.expanduser(options.cache)
|
||||
|
||||
eos.config.gamedata_connectionstring = options.dump
|
||||
eos.config.debug = False
|
||||
|
||||
from eos.gamedata import *
|
||||
import eos.db
|
||||
|
||||
# Get version of EVE client
|
||||
config = ConfigParser()
|
||||
config.read(os.path.join(PATH_EVE, "common.ini"))
|
||||
|
||||
# Form metadata dictionary for corresponding table
|
||||
metadata = {}
|
||||
metadata["version"] = config.getint("main", "build")
|
||||
metadata["release"] = options.release
|
||||
|
||||
# Initialize Reverence cache manager
|
||||
eve = blue.EVE(PATH_EVE, cachepath=PATH_CACHE, server=server)
|
||||
cfg = eve.getconfigmgr()
|
||||
|
||||
# Create all tables we need
|
||||
eos.db.gamedata_meta.create_all()
|
||||
|
||||
# Add versioning info to the metadata table
|
||||
for fieldname in metadata:
|
||||
eos.db.gamedata_session.add(MetaData(fieldname, metadata[fieldname]))
|
||||
|
||||
eos.db.gamedata_session.commit()
|
||||
|
||||
# Get table map, processing order and special table data
|
||||
TABLE_MAP = get_map()
|
||||
TABLE_ORDER = get_order()
|
||||
CUSTOM_CALLS = get_customcalls()
|
||||
|
||||
# Warn about various stuff
|
||||
for table in cfg.tables:
|
||||
if not table in TABLE_MAP:
|
||||
# Warn about new tables in cache which are still not described by table map
|
||||
print "Warning: unmapped table {0} found in cache".format(table)
|
||||
for table in TABLE_MAP:
|
||||
if not table in cfg.tables and not table in CUSTOM_CALLS:
|
||||
# Warn about mapped tables which are missing in cache
|
||||
print "Warning: mapped table {0} cannot be found in cache".format(table)
|
||||
if not table in TABLE_ORDER and TABLE_MAP[table] is not None:
|
||||
# Warn about mapped tables not specified in processing order
|
||||
print "Warning: mapped table {0} is missing in processing order".format(table)
|
||||
for table in TABLE_ORDER:
|
||||
if not table in TABLE_MAP:
|
||||
# Warn about unmapped tables in processing order
|
||||
print "Warning: unmapped table {0} is specified in processing order".format(table)
|
||||
|
||||
# Get data from reverence and write it
|
||||
for tablename in TABLE_ORDER:
|
||||
tableclass = TABLE_MAP[tablename]
|
||||
if tableclass is not None:
|
||||
# Print currently processed table name
|
||||
print "Processing: {0}".format(tablename)
|
||||
# Get table object from the Reverence and process it
|
||||
source_table = getattr(cfg, tablename) if tablename not in CUSTOM_CALLS else CUSTOM_CALLS[tablename]
|
||||
# Gather data regarding columns for current table in cache and eos
|
||||
cols_eos = set(prop.key for prop in class_mapper(TABLE_MAP[tablename]).iterate_properties if isinstance(prop, ColumnProperty))
|
||||
cols_rev = set(get_source_headers(source_table))
|
||||
notineos = cols_rev.difference(cols_eos)
|
||||
notinrev = cols_eos.difference(cols_rev)
|
||||
if notineos:
|
||||
print "Warning: {0} found in cache but missing in eos definitions: {1}".format("column" if len(notineos) == 1 else "columns", ", ".join(sorted(notineos)))
|
||||
if notinrev:
|
||||
print "Warning: {0} found in eos definitions but missing in cache: {1}".format("column" if len(notinrev) == 1 else "columns", ", ".join(sorted(notinrev)))
|
||||
process_table(source_table, tablename, tableclass)
|
||||
Reference in New Issue
Block a user