Prepare database

1. extract recipes
2. download images
3. extract localized titles

make-db/main.py

@@ -0,0 +1,96 @@
+from time import sleep
+import sqlite3
+import xml.etree.ElementTree as ElementTree
+import typer
+import requests
+
+
+base_url = "https://render.albiononline.com/v1/item/{uniquename}.png"
+
+app = typer.Typer()
+
+def load_xml(filename: str):
+    tree = ElementTree.parse(filename)
+    return tree
+
+def filter_recipes(items):
+    items_all = set()
+    recipes = []
+    a = items.findall("consumableitem")
+    for item in a:
+        un = item.attrib["uniquename"]
+        if "UNIQUE" in un or "VANITY" in un:
+            print(un)
+            continue
+        try:
+            items_all.add(un)
+            t = item.attrib["slottype"]
+            for_item = []
+            for child in item:
+                if child.tag == "craftingrequirements":
+                    am = child.attrib["amountcrafted"]
+                    res = {r.attrib["uniquename"]: r.attrib["count"] for r in child}
+                    items_all.update(res.keys())
+                    for_item.append((am, res))
+            recipes.append((un, t, for_item))
+        except Exception as e:
+            print(e, un)
+    return items_all, recipes
+
+def save_items(items_all, db):
+    db.execute("DROP TABLE IF EXISTS item;")
+    db.execute("CREATE TABLE item (uniquename STRING);")
+    db.executemany("INSERT INTO item VALUES (?);", map(lambda x: (x,), items_all))
+
+def save_recipes(recipes, db):
+    db.execute("DROP TABLE IF EXISTS recipe;")
+    db.execute("CREATE TABLE recipe (item INTEGER REFERENCES item(ROWID), amountcrafted INTEGER);")
+    db.execute("DROP TABLE IF EXISTS ingredient;")
+    db.execute("CREATE TABLE ingredient (recipe INTEGER REFERENCES recipe(ROWID), item INTEGER REFERENCES item(ROWID), amount INTEGER);")
+    db = db.cursor()
+    for un, t, for_item in recipes:
+        for am, res in for_item:
+            db.execute("INSERT INTO recipe VALUES ((SELECT ROWID FROM item WHERE uniquename=?), ?);", (un, am))
+            i = db.lastrowid
+            ingredients = map(lambda x: (i, x[0], x[1]), res.items())
+            db.executemany("INSERT INTO ingredient VALUES (?, (SELECT ROWID FROM item WHERE uniquename=?), ?);", ingredients)
+
+@app.command()
+def extract_recipes(path_to_items_xml: str, output_filename: str):
+    items = load_xml(path_to_items_xml)
+    items_all, recipes = filter_recipes(items)
+    with sqlite3.connect(output_filename) as db:
+        save_items(items_all, db)
+        save_recipes(recipes, db)
+
+@app.command()
+def download_images(path_to_db: str):
+    with sqlite3.connect(path_to_db) as db:
+        db.execute("DROP TABLE IF EXISTS image;")
+        db.execute("CREATE TABLE image (item INTEGER REFERENCES item(ROWID), image BLOB);")
+        for i, uniquename in db.execute("SELECT ROWID, uniquename FROM item;"):
+            response = requests.get(base_url.format(uniquename=uniquename))
+            if response.status_code != 200:
+                print(uniquename)
+                continue
+            db.execute("INSERT INTO image VALUES (?, ?);", (i, response.content))
+            sleep(1)
+
+@app.command()
+def extract_titles(path_to_localization_xml: str, path_to_db: str):
+    with sqlite3.connect(path_to_db) as db:
+        db.execute("DROP TABLE IF EXISTS title;")
+        db.execute("CREATE TABLE title (item INTEGER REFERENCES item(ROWID), en STRING, ru STRING);")
+        uns = dict(db.execute("SELECT uniquename, ROWID FROM item;").fetchall())
+        tree = ElementTree.parse(path_to_localization_xml)
+        ns = {"xml":"http://www.w3.org/XML/1998/namespace"}
+        for tu in tree.findall("body/tu"):
+            un = tu.attrib["tuid"].removeprefix("@ITEMS_")
+            if un in uns:
+                en = tu.find("tuv[@xml:lang='EN-US']/seg", ns).text
+                ru = tu.find("tuv[@xml:lang='RU-RU']/seg", ns).text
+                db.execute("INSERT INTO title VALUES (?, ?, ?);", (uns[un], en, ru))
+
+
+if __name__ == "__main__":
+    app()