User:OmegaK2/cleanup

init stuff

import os
import shutil
import mwclient
import re
from PyPoE.poe.sim.mods import get_translation
from PyPoE.poe.constants import *
from PyPoE.poe.file import dat
from PyPoE.poe.file.ot import OTFileCache
from PyPoE.poe.file.ggpk import GGPKFile, extract_dds
from PyPoE.poe.file.translations import TranslationFileCache, MissingIdentifierWarning, install_data_dependant_quantifiers
from collections import OrderedDict, defaultdict
site = mwclient.Site(('https', 'pathofexile.gamepedia.com'), path='/')


spec = dat.set_default_spec(version=VER)

#
# Some global data I need to use 
#

c_weapons = 'Claws', 'Daggers', 'Wands', 'One Hand Swords', 'Thrusting One Hand Swords', 'One Hand Axes', 'One Hand Maces', 'Bows', 'Staves', 'Two Hand Swords', 'Two Hand Maces', 'Two Hand Axes', 'Sceptres', 'Fishing Rods'
c_armour = 'Boots', 'Helmets', 'Gloves', 'Shields', 'Body Armours'
c_flasks = 'Utility Flasks', 'Critical Utility Flasks', 'Life Flasks', 'Mana Flasks', 'Hybrid Flasks'
c_gems = 'Active Skill Gems', 'Support Skill Gems',
c_jewellery = 'Amulets', 'Belts', 'Rings'

#
#
#

path = 'C:/Temp/'

instopt = {
}

opt = {
    'use_dat_value': False,
    'auto_build_index': True,
}
r = dat.RelationalReader(
    path_or_ggpk=path,
    raise_error_on_missing_relation=False,
    instance_options=instopt,
    read_options=opt)
#r = dat.RelationalReader(path_or_ggpk=path, read_options=opt)
#
install_data_dependant_quantifiers(r)
tc = TranslationFileCache(path_or_ggpk=path, files=['stat_descriptions.txt'], merge_with_custom_file=False)
ot = OTFileCache(path_or_ggpk=path)

Cleanup old data via id

class Wiki:
    @staticmethod
    def fix_mods():
        offset = '0'
        rows = {}
        while offset:
            results=site.raw_api(
                'cargoquery',
                fields='mods._pageName=page, mods.id=id',
                tables='mods',
                limit='5000',
                offset=offset,
            )['cargoquery']

            for row in results:
                row = row['title']
                rows[row['id']] = row['page']

            if len(results) == 5000:
                offset = str(int(offset) + 5000)
            else:
                offset = None

        extra = []

        for row in r['Mods.dat']:
            try:
                del rows[row['Id']]
            except KeyError:
                extra.append(row['Id'])

        print(len(rows))
        print(len(extra))