Source code for core.mods

#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Mod Pack management and merging tools."""

import glob
import os
import shutil
import sys
import time
from difflib import SequenceMatcher, ndiff

from . import baselines, log, manifest, paths
from .lnp import lnp


def _shutil_wrap(fn):
    def _wrapped_fn(*args, **kwargs):
        i = 0
        while i < 5:
            try:
                fn(*args, **kwargs)
            except Exception:
                i += 1
                time.sleep(0.1)
            else:
                break
    return _wrapped_fn


# Use sys.platform directly to prevent possible issues with initialization order
if sys.platform == 'win32':
    shutil.rmtree = _shutil_wrap(shutil.rmtree)
    shutil.copytree = _shutil_wrap(shutil.copytree)


[docs]def toggle_premerge_gfx(): """Sets the option for pre-merging of graphics.""" lnp.userconfig['premerge_graphics'] = not lnp.userconfig.get_bool( 'premerge_graphics') lnp.userconfig.save_data()
[docs]def will_premerge_gfx(): """Returns whether graphics will be merged prior to any mods.""" return lnp.userconfig.get_bool('premerge_graphics')
[docs]def read_mods(): """Returns a list of mod packs""" return [os.path.basename(o) for o in glob.glob(paths.get('mods', '*')) if os.path.isdir(o) and manifest.is_compatible('mods', os.path.basename(o))]
[docs]def get_title(mod): """Returns the mod title; either per manifest or from dirname.""" title = manifest.get_cfg('mods', mod).get_string('title') if title: return title return mod
[docs]def get_tooltip(mod): """Returns the tooltip for the given mod.""" return manifest.get_cfg('mods', mod).get_string('tooltip')
[docs]def simplify_mods(): """Removes unnecessary files from all mods.""" mods, files = 0, 0 for pack in read_mods(): mods += 1 files += simplify_pack(pack) return mods, files
[docs]def simplify_pack(pack): """Removes unnecessary files from one mod. Params: pack path segment in './LNP/Mods/pack/' as a string Returns: The sum of files affected by the operations """ # Here we use the heuristic that mods which are bundled with other files # contain a complete set of raws, and vanilla files which are missing # should not be inserted. We thus add empty files to fill out the set in # cases where several files are removed. i = baselines.simplify_pack(pack, 'mods') if not i: i = 0 if i > 10: log.w('Reducing mod "{}": assume vanilla files were omitted ' 'deliberately'.format(pack)) i += make_blank_files(pack) i += baselines.remove_vanilla_raws_from_pack(pack, 'mods') i += baselines.remove_empty_dirs(pack, 'mods') return i
[docs]def make_blank_files(pack): """Create blank files where vanilla files are missing. Params: pack path segment in './LNP/folder/pack/' as strings Returns: The number of blank files created """ i = 0 vanilla_raws = baselines.find_vanilla_raws() for root, _, files in os.walk(vanilla_raws): for k in files: f = os.path.relpath(os.path.join(root, k), vanilla_raws) if not os.path.isfile(paths.get('mods', pack, f)): filepath = paths.get('mods', pack, f) with open(filepath, 'w', encoding="utf-8") as blank: blank.write('') i += 1 return i
[docs]def install_mods(): """Deletes installed raw folder, and copies over merged raws.""" merge_log = paths.get('baselines', 'temp', 'raw', 'installed_raws.txt') if read_installation_log(merge_log): shutil.rmtree(paths.get('df', 'raw')) shutil.rmtree(paths.get('df', 'data', 'speech')) shutil.copytree(paths.get('baselines', 'temp', 'raw'), paths.get('df', 'raw')) shutil.copytree(paths.get('baselines', 'temp', 'data', 'speech'), paths.get('df', 'data', 'speech')) return True log.w('To avoid data loss, PyLNP only installs mods if a log exists') return False
[docs]def merge_all_mods(list_of_mods, gfx=None): """Merges the specified list of mods, starting with graphics if set to pre-merge (or if a pack is specified explicitly). Params: list_of_mods a list of the names of mods to merge gfx a graphics pack to be merged in Returns: A list of status ints for each mod given: -1: Unmerged 0: Merge was successful, all well 1: Potential compatibility issues, no merge problems 2: Non-fatal error, overlapping lines or non-existent mod etc. 3: Fatal error, not returned (rebuilds to previous, rest unmerged) """ from . import graphics clear_temp() if gfx: add_graphics(gfx) elif will_premerge_gfx(): add_graphics(graphics.current_pack()) ret_list = [] for i, mod in enumerate(list_of_mods): status = merge_a_mod(mod) ret_list.append(status) if status == 3: log.i('Mod {}, in {}, could not be merged.'.format( mod, str(list_of_mods))) merged = merge_all_mods(list_of_mods[:i - 1], gfx) return merged + [-1] * len(list_of_mods[i:]) return ret_list
[docs]def merge_a_mod(mod): """Merges the specified mod, and returns an exit code 0-3. 0: Merge was successful, all well 1: Potential compatibility issues, no merge problems 2: Non-fatal error, overlapping lines or non-existent mod etc. 3: Fatal error, respond by rebuilding to previous mod """ log.push_prefix('In "' + mod + '": ') if not baselines.find_vanilla_raws(): log.e('Could not merge: baseline raws unavailable') return 3 log.d('Starting to merge mod: {}'.format(mod)) mod_raw_folder = paths.get('mods', mod, 'raw') if not os.path.isdir(mod_raw_folder): log.w('mod is invalid; /raw/ must be a directory') return 2 status = merge_folder(mod_raw_folder, baselines.find_vanilla_raws(), paths.get('baselines', 'temp', 'raw')) if os.path.isdir(paths.get('mods', mod, 'data', 'speech')): status = max(status, merge_folder( paths.get('mods', mod, 'data', 'speech'), os.path.join(baselines.find_vanilla(), 'data', 'speech'), paths.get('baselines', 'temp', 'data', 'speech'))) if status < 3: with open(paths.get('baselines', 'temp', 'raw', 'installed_raws.txt'), 'a', encoding="utf-8") as f: f.write('mods/' + mod + '\n') log.i('Finished merging') log.pop_prefix() return status
[docs]def merge_folder(mod_folder, vanilla_folder, mixed_folder): """Merge the specified folders, output going in 'LNP/Baselines/temp' Text files are merged; other files (sprites etc.) are copied over.""" status = 0 for root, _, files in os.walk(mod_folder): # We want to make any directory in our mod folder in the mixed folder # if it doesn't already exist. Fixes #173 mixed_dir = os.path.join(mixed_folder, os.path.relpath(root, mod_folder)) if not os.path.isdir(mixed_dir): os.makedirs(mixed_dir) for k in files: f = os.path.relpath(os.path.join(root, k), mod_folder) log.push_prefix('file "' + f + '": ') log.d('merging...') mod_f = os.path.join(mod_folder, f) van_f = os.path.join(vanilla_folder, f) gen_f = os.path.join(mixed_folder, f) if any(f.endswith(a) for a in ('.txt', '.init')): # merge raws and DFHack init files status = max(status, merge_file(mod_f, van_f, gen_f)) elif any(f.endswith(a) for a in ('.lua', '.rb', '.bmp', '.png')): # copy DFHack scripts or sprite sheets if not os.path.isdir(os.path.dirname(gen_f)): os.makedirs(os.path.dirname(gen_f)) if not os.path.isfile(gen_f): shutil.copy2(mod_f, gen_f) status = max(1, status) else: with open(mod_f, 'rb') as f: mb = f.read() with open(gen_f, 'rb') as f: gb = f.read() if mb != gb: shutil.copyfile(mod_f, gen_f) status = max(2, status) log.d('merged with status {}'.format(status)) log.pop_prefix() return status
[docs]def merge_file(mod_file_name, van_file_name, gen_file_name): """Merges three files, and returns an exit code 0-3. 0: Merge was successful, all well 1: Potential compatibility issues, no merge problems 2: Non-fatal error, overlapping lines or non-existent mod etc. 3: Fatal error, respond by rebuilding to previous mod """ van_lines, mod_lines, gen_lines = [], [], [] for fname, lines in ((van_file_name, van_lines), (mod_file_name, mod_lines), (gen_file_name, gen_lines)): try: with open(fname, encoding='cp437', errors='replace') as f: lines.extend(f.readlines()) except IOError: log.d(fname + ' cannot be read; merging other files') status, gen_lines = merge_line_list(mod_lines, van_lines, gen_lines) try: with open(gen_file_name, "w", encoding='cp437') as gen_file: gen_file.writelines(gen_lines) except Exception: log.e('Writing to {} failed'.format(gen_file_name)) status = 3 return status
[docs]def merge_line_list(mod_text, vanilla_text, gen_text): """Merges sequences of lines. Params: mod_text The lines of the mod file being added to the merge. vanilla_text The lines of the corresponding vanilla file. gen_text The lines of the previously merged file or files. Returns: tuple(status, lines); status is 0/'ok' or 2/'overlap merged' """ if mod_text and vanilla_text == gen_text: log.d('no overlap with previous mods, replacing vanilla file') return 0, mod_text if gen_text and vanilla_text == mod_text: log.d('mod file identical to vanilla file') return 0, gen_text if gen_text and gen_text == mod_text: log.d('changes are identical to a previously merged mod') return 0, gen_text if mod_text and gen_text and not vanilla_text: log.d('Falling back to two-way merge; no vanilla file exists.') return 0, [s[2:] for s in ndiff(gen_text, mod_text)] log.d('performing three-way merge') # SequenceMatcher describes the diff to vanilla gen_ops = SequenceMatcher(None, vanilla_text, gen_text).get_opcodes() mod_ops = SequenceMatcher(None, vanilla_text, mod_text).get_opcodes() outfile = [] for block in three_way_merge(gen_text, gen_ops, mod_text, mod_ops): log.d('writing block') outfile.extend(block) status = outfile.pop() return status, outfile
[docs]def three_way_merge(gen_text, van_gen_ops, mod_text, van_mod_ops): """Yield blocks of lines from a three-way-merge. Last block is status.""" # pylint:disable=too-many-statements status, cur_v, mod_i2, gen_i2 = 0, 0, 1, 1 while van_mod_ops and van_gen_ops: log.d('before pop') log.d('gen ops: ' + str(van_gen_ops)) log.d('mod ops: ' + str(van_mod_ops)) log.d( 'mod_i2: ' + str(mod_i2) + ' gen_i2: ' + str(gen_i2) + ' cur_v: ' + str(cur_v)) if mod_i2 <= cur_v: log.d('pop mod') van_mod_ops.pop(0) if gen_i2 <= cur_v: log.d('pop gen') van_gen_ops.pop(0) log.d('after pop') log.d('gen ops: ' + str(van_gen_ops)) log.d('mod ops: ' + str(van_mod_ops)) if len(van_mod_ops) == 0 or len(van_gen_ops) == 0: log.d('out of entries') break _, _, mod_i2, mod_j1, mod_j2 = van_mod_ops[0] gen_tag, _, gen_i2, gen_j1, gen_j2 = van_gen_ops[0] low_i2 = min(mod_i2, gen_i2) if van_mod_ops[0][0] == 'equal': log.d('equal mod ops') if gen_tag == 'equal': log.d('equal gen tag') yield gen_text[cur_v:low_i2] cur_v = low_i2 log.d('back from equal gen tag') continue log.d('not equal gen tag') yield gen_text[gen_j1:gen_j2] cur_v = gen_i2 continue if gen_tag == 'equal': log.d('not equal mod op, equal gen tag') yield mod_text[mod_j1:mod_j2] cur_v = mod_i2 continue log.d( 'yield mod text (cur_v: ' + str(cur_v) + ', mod_j2: ' + str(mod_j2) + '): ' + str(mod_text[cur_v:mod_j2])) yield mod_text[cur_v:mod_j2] log.d('back from yield mod text') if gen_text[cur_v:low_i2] != mod_text[cur_v:low_i2]: status = 2 log.d('Overwrite merge at line {}'.format(cur_v)) log.v('- ' + '- '.join(gen_text[cur_v:low_i2]) + '+ ' + '+ '.join(mod_text[cur_v:low_i2])) cur_v = low_i2 while van_mod_ops: log.d('popping mod ops') _, _, _, mod_j1, mod_j2 = van_mod_ops.pop(0) yield mod_text[mod_j1:mod_j2] while van_gen_ops: log.d('popping gen ops') _, _, _, gen_j1, gen_j2 = van_gen_ops.pop(0) yield gen_text[gen_j1:gen_j2] log.d('yield status') yield [status]
[docs]def clear_temp(): """Resets the folder in which raws are mixed.""" if not baselines.find_vanilla_raws(False): log.e('Could not clear temp: baseline raws unavailable') return if os.path.exists(paths.get('baselines', 'temp')): shutil.rmtree(paths.get('baselines', 'temp')) shutil.copytree(baselines.find_vanilla_raws(), paths.get('baselines', 'temp', 'raw')) shutil.rmtree(paths.get('baselines', 'temp', 'raw', 'graphics')) shutil.copytree(os.path.join(baselines.find_vanilla(), 'data', 'speech'), paths.get('baselines', 'temp', 'data', 'speech')) with open(paths.get('baselines', 'temp', 'raw', 'installed_raws.txt'), 'w', encoding="utf-8") as f: f.write('# List of raws merged by PyLNP:\nbaselines/' + os.path.basename(baselines.find_vanilla()) + '\n')
[docs]def update_raw_dir(path, gfx=('', '')): """Updates a raw dir in place with specified graphics and raws. Returns: True if completed, or False if aborted. Arguments: path the full path to the dir to update gfx Tuple of graphics pack to update to, and pack installed in baselines/temp/ """ mods_list = read_installation_log(os.path.join(path, 'installed_raws.txt')) built_log = paths.get('baselines', 'temp', 'raw', 'installed_raws.txt') built_mods = read_installation_log(built_log) if mods_list != built_mods or gfx[0] != gfx[1]: if -1 in merge_all_mods(mods_list, gfx[0]): log.w('Some mods in {} could not be re-merged'.format(path)) return False shutil.rmtree(path) shutil.copytree(paths.get('baselines', 'temp', 'raw'), path) return True
[docs]def add_graphics(gfx): """Adds graphics to the mod merge in baselines/temp.""" from . import graphics gfx_raws = paths.get('graphics', gfx, 'raw') for root, _, files in os.walk(gfx_raws): dst = paths.get('baselines', 'temp', 'raw', os.path.relpath(root, gfx_raws)) if not os.path.isdir(dst): os.makedirs(dst) for f in files: shutil.copy2(os.path.join(root, f), dst) with open(paths.get('baselines', 'temp', 'raw', 'installed_raws.txt'), 'a', encoding="utf-8") as f: f.write('graphics/{}\n'.format(graphics.get_folder_prefix(gfx))) log.i('{} graphics added (small mod compatibility risk)'.format(gfx))
[docs]def can_rebuild(log_file, strict=True): """Test if user can exactly rebuild a raw folder, returning a bool.""" if not os.path.isfile(log_file): guess = not strict log.w('{} not found; assume rebuildable = {}'.format(log_file, guess)) return guess mod_list = read_installation_log(log_file) return all(m in read_mods() for m in mod_list)
[docs]def make_mod_from_installed_raws(name): """Capture whatever unavailable mods a user currently has installed as a mod called $name. * If ``installed_raws.txt`` is not present, compare to vanilla * Otherwise, rebuild as much as possible, then compare to installed """ if get_installed_mods_from_log(): clear_temp() for mod in get_installed_mods_from_log(): merge_a_mod(mod) reconstruction = paths.get('baselines', 'temp2') shutil.copytree(paths.get('baselines', 'temp'), reconstruction) else: reconstruction = baselines.find_vanilla() if not reconstruction: return None clear_temp() merge_folder(os.path.join(reconstruction, 'raw'), paths.get('df', 'raw'), paths.get('baselines', 'temp', 'raw')) merge_folder(os.path.join(reconstruction, 'data', 'speech'), paths.get('df', 'data', 'speech'), paths.get('baselines', 'temp', 'data', 'speech')) baselines.simplify_pack('temp', 'baselines') baselines.remove_vanilla_raws_from_pack('temp', 'baselines') baselines.remove_empty_dirs('temp', 'baselines') if os.path.isdir(paths.get('baselines', 'temp2')): shutil.rmtree(paths.get('baselines', 'temp2')) if name and os.path.isdir(paths.get('baselines', 'temp')): if os.path.isdir(paths.get('mods', name)): return False shutil.copytree(paths.get('baselines', 'temp'), paths.get('mods', name)) return True return None
[docs]def get_installed_mods_from_log(): """Return best mod load order to recreate installed with available.""" logged = read_installation_log(paths.get('df', 'raw', 'installed_raws.txt')) return [mod for mod in logged if mod in read_mods()]
[docs]def read_installation_log(fname): """Read an 'installed_raws.txt' and return the mods.""" try: with open(fname, encoding="utf-8") as f: file_contents = list(f.readlines()) except IOError: log.d('Log not found: ' + fname) return [] mods_list = [] for line in file_contents: if line.startswith('mods/'): mods_list.append(line.strip().replace('mods/', '')) return mods_list