diff options
Diffstat (limited to 'updater')
| -rw-r--r-- | updater/MT_skins_updater.cs | 200 | ||||
| -rw-r--r-- | updater/MT_skins_updater.exe | bin | 0 -> 10752 bytes | |||
| -rw-r--r-- | updater/Newtonsoft.Json.dll | bin | 0 -> 491008 bytes | |||
| -rwxr-xr-x | updater/update_from_db.py | 180 | ||||
| -rwxr-xr-x | updater/update_skins_db.sh | 79 |
5 files changed, 459 insertions, 0 deletions
diff --git a/updater/MT_skins_updater.cs b/updater/MT_skins_updater.cs new file mode 100644 index 0000000..6d27da0 --- /dev/null +++ b/updater/MT_skins_updater.cs @@ -0,0 +1,200 @@ +using System; +//Json.NET library (http://json.codeplex.com/) +using Newtonsoft.Json; +using System.Collections.Generic; +using System.Net; +using System.IO; + +// MT skins updater for the u_skins mod +// Creator: Krock +// License: zlib (http://www.zlib.net/zlib_license.html) +namespace MT_skins_updater { + class Program { + static void Main(string[] args) { + Console.WriteLine("Welcome to the MT skins updater!"); + Console.WriteLine("# Created by: Krock (2014-07-10)"); + Engine e = new Engine(); + Console.WriteLine(@"Path to the u_skins mod: (ex. 'E:\Minetest\mods\u_skinsdb\u_skins\')"); + string path = Console.ReadLine(); + Console.WriteLine("Start updating at page: ('0' to update everything)"); + int page = getInt(Console.ReadLine()); + e.Start(path, page); + Console.WriteLine("Press any key to exit."); + Console.ReadKey(false); + } + public static int getInt(string i) { + int ret = 0; + int.TryParse(i, out ret); + return (ret > 0)? ret : 0; + } + } + class Engine { + string root = "http://minetest.fensta.bplaced.net"; + bool alternate = true; //should it use the special version of medadata saving? + + public void Start(string path, int page) { + if (path.Length < 5) { + Console.WriteLine("Too short path. STOP."); + return; + } + if (path[path.Length - 1] != '\\') { + path += '\\'; + } + if(!Directory.Exists(path + "meta")){ + Console.WriteLine("Folder 'meta' not found. STOP."); + return; + } + if(!Directory.Exists(path + "textures")){ + Console.WriteLine("Folder 'textures' not found. STOP."); + return; + } + WebClient cli = new WebClient(); + //add useragent to identify + cli.Headers.Add("User-Agent", "MT_skin_grabber 1.1"); + + bool firstSkin = true; + List<string> skin_local = new List<string>(); + int pages = page, + updated = 0; + + for (; page <= pages; page++) { + string contents = ""; + try { + contents = cli.DownloadString(root + "/api/get.json.php?getlist&page=" + page); + } catch(WebException e) { + Console.WriteLine("Whoops! Error at page ID: " + page + ". WebClient sais: " + e.Message); + Console.WriteLine("Press any key to skip this page."); + Console.ReadKey(false); + continue; + } + Data o = JsonConvert.DeserializeObject<Data>(contents); + if (o.pages != pages) { + pages = o.pages; + } + + Console.WriteLine("# Page " + page + " (" + o.per_page + " skins)"); + for (int i = 0; i < o.skins.Length; i++) { + int id = o.skins[i].id; + if(o.skins[i].type != "image/png"){ + Console.WriteLine("Image type '" + o.skins[i].type + "' not supported at skin ID: " + id); + Console.WriteLine("Press any key to continue."); + Console.ReadKey(false); + continue; + } + //eliminate special chars! + o.skins[i].name = WebUtility.HtmlDecode(o.skins[i].name); + o.skins[i].author = WebUtility.HtmlDecode(o.skins[i].author); + + //to delete old, removed skins + if (firstSkin) { + firstSkin = false; + + string[] files = Directory.GetFiles(path + "textures\\"); + for (int f = 0; f < files.Length; f++) { + string[] filePath = stringSplitLast(files[f], '\\'), + fileName = stringSplitLast(filePath[1], '.'), + fileVer = stringSplitLast(fileName[0], '_'); + if (fileVer[1] == "" || fileVer[0] != "character") continue; + + int skinNr = Program.getInt(fileVer[1]); + if (skinNr <= id) continue; + skin_local.Add(fileName[0]); + } + } else skin_local.Remove("character_" + id); + + //get file size, only override changed + FileInfo localImg = new FileInfo(path + "textures\\character_" + id + ".png"); + byte[] imageData = Convert.FromBase64String(o.skins[i].img); + bool isDif = true; + if (localImg.Exists) isDif = (Math.Abs(imageData.Length - localImg.Length) >= 3); + + if (isDif) { + File.WriteAllBytes(localImg.FullName, imageData); + imageData = null; + //previews + try { + cli.DownloadFile(root + "/skins/1/" + id + ".png", path + "textures\\character_" + id + "_preview.png"); + } catch (WebException e) { + Console.WriteLine("Whoops! Error at skin ID: " + id + ". WebClient sais: " + e.Message); + Console.WriteLine("Press any key to continue."); + Console.ReadKey(false); + } + } else { + Console.WriteLine("[SKIP] character_" + id); + continue; + } + + string meta = ""; + if (!alternate) { + meta = "name = \"" + o.skins[i].name + "\",\n"; + meta += "author = \"" + o.skins[i].author + "\",\n"; + meta += "comment = \"" + o.skins[i].license + '"'; + } else { + meta = o.skins[i].name + '\n' + o.skins[i].author + '\n' + o.skins[i].license; + } + File.WriteAllText(path + "meta\\character_" + id + ".txt", meta); + updated++; + Console.WriteLine("[" + id + "] " + shorten(o.skins[i].name, 20) + "\t by: " + o.skins[i].author + "\t (" + o.skins[i].license + ")"); + } + } + foreach (string fileName in skin_local) { + if(File.Exists(path + "textures\\" + fileName + ".png")) { + File.Delete(path + "textures\\" + fileName + ".png"); + } + if(File.Exists(path + "textures\\" + fileName + "_preview.png")) { + File.Delete(path + "textures\\" + fileName + "_preview.png"); + } + if(File.Exists(path + "meta\\" + fileName + ".txt")) { + File.Delete(path + "meta\\" + fileName + ".txt"); + } + Console.WriteLine("[DEL] " + fileName + " (deleted skin)"); + } + Console.WriteLine("Done. Updated " + updated + " skins!"); + } + string shorten(string inp, int len) { + char[] shr = new char[len]; + for (int i = 0; i < len; i++) { + if (i < inp.Length) { + shr[i] = inp[i]; + } else shr[i] = ' '; + } + return new string(shr); + } + + string[] stringSplitLast(string path, char limiter) { + int found = 0; + int totalLen = path.Length - 1; + for (int i = totalLen; i >= 0; i--) { + if (path[i] == limiter) { + found = i; + break; + } + } + if (found == 0) { + return new string[] { "", "" }; + } + + int len = totalLen - found; + char[] str_1 = new char[found], + str_2 = new char[len]; + + for (int i = 0; i < path.Length; i++) { + if (i == found) continue; + if (i < found) { + str_1[i] = path[i]; + } else { + str_2[i - found - 1] = path[i]; + } + } + return new string[] { new string(str_1), new string(str_2) }; + } + } + class Data { + public Skins_data[] skins; + public int page, pages, per_page; + } + class Skins_data { + public string name, author, uploaded, type, license, img; + public int id, license_id; + } +} diff --git a/updater/MT_skins_updater.exe b/updater/MT_skins_updater.exe Binary files differnew file mode 100644 index 0000000..5b4ee3e --- /dev/null +++ b/updater/MT_skins_updater.exe diff --git a/updater/Newtonsoft.Json.dll b/updater/Newtonsoft.Json.dll Binary files differnew file mode 100644 index 0000000..054c933 --- /dev/null +++ b/updater/Newtonsoft.Json.dll diff --git a/updater/update_from_db.py b/updater/update_from_db.py new file mode 100755 index 0000000..685d1e0 --- /dev/null +++ b/updater/update_from_db.py @@ -0,0 +1,180 @@ +#!/usr/bin/python3 +from http.client import HTTPConnection,HTTPException,BadStatusLine,_CS_IDLE +import json +import base64 +from contextlib import closing +import sys,os,shutil,time + +def die(message,code=23): + print(message,file=sys.stderr) + raise SystemExit(code) + +server = "minetest.fensta.bplaced.net" +skinsdir = "../textures/" +metadir = "../meta/" +curskin = 0 +curpage = 1 +pages = None + +def replace(location,base,encoding=None,path=None): + if path is None: + path = os.path.join(location,base) + mode = "wt" if encoding else "wb" + # an unpredictable temp name only needed for a+rwxt directories + tmp = os.path.join(location,'.'+base+'-tmp') + def deco(handle): + with open(tmp,mode,encoding=encoding) as out: + handle(out) + os.rename(tmp,path) + return deco + +def maybeReplace(location,base,encoding=None): + def deco(handle): + path = os.path.join(location,base) + if os.path.exists(path): return + return replace(location,base,encoding=encoding,path=path)(handle) + return deco + +class Penguin: + "idk" + def __init__(self, url, recv, diemessage): + self.url = url + self.recv = recv + self.diemessage = diemessage + +class Pipeline(list): + "Gawd why am I being so elaborate?" + def __init__(self, threshold=10): + "threshold is how many requests in parallel to pipeline" + self.threshold = threshold + self.sent = True + def __enter__(self): + self.reopen() + return self + def __exit__(self,typ,exn,trace): + self.send() + self.drain() + def reopen(self): + self.c = HTTPConnection(server) + self.send() + def append(self,url,recv,diemessage): + self.sent = False + super().append(Penguin(url,recv,diemessage)) + if len(self) > self.threshold: + self.send() + self.drain() + def trydrain(self): + for penguin in self: + print('drain',penguin.url) + try: + penguin.response.begin() + penguin.recv(penguin.response) + except BadStatusLine as e: + print('derped requesting',penguin.url) + return False + except HTTPException as e: + die(penguin.diemessage+' '+repr(e)+' (url='+penguin.url+')') + self.clear() + return True + def drain(self): + print('draining pipeline...',len(self)) + assert self.sent, "Can't drain without sending the requests!" + self.sent = False + while self.trydrain() is not True: + self.c.close() + print('drain failed, trying again') + time.sleep(1) + self.reopen() + def trysend(self): + for penguin in pipeline: + print('fill',penguin.url) + try: + self.c.request("GET", penguin.url) + self.c._HTTPConnection__state = _CS_IDLE + penguin.response = self.c.response_class(self.c.sock, + method="GET") + # begin LATER so we can send multiple requests w/out response headers + except BadStatusLine: + return False + except HTTPException as e: + die(diemessage+' because of a '+repr(e)) + return True + def send(self): + if self.sent: return + print('filling pipeline...',len(self)) + while self.trysend() is not True: + self.c.close() + print('derped resending') + time.sleep(1) + self.reopen() + self.sent = True + +with Pipeline() as pipeline: + # two connections is okay, right? one for json, one for preview images + c = HTTPConnection(server) + def addpage(page): + global curskin, pages + print("Page: " + str(page)) + r = 0 + try: + c.request("GET", "/api/get.json.php?getlist&page=" + str(page) + "&outformat=base64") + r = c.getresponse() + except Exception: + if r != 0: + if r.status != 200: + die("Error", r.status) + return + + data = r.read().decode() + l = json.loads(data) + if not l["success"]: + die("Success != True") + r = 0 + pages = int(l["pages"]) + foundOne = False + for s in l["skins"]: + # make sure to increment this, even if the preview exists! + curskin = curskin + 1 + previewbase = "character_" + str(curskin) + "_preview.png" + preview = os.path.join(skinsdir, previewbase) + if os.path.exists(preview): + print('skin',curskin,'already retrieved') + continue + print('updating skin',curskin,'id',s["id"]) + foundOne = True + @maybeReplace(skinsdir, "character_" + str(curskin) + ".png") + def go(f): + f.write(base64.b64decode(bytes(s["img"], 'utf-8'))) + f.close() + + @maybeReplace(metadir, "character_" + str(curskin) + ".txt", + encoding='utf-8') + def go(f): + f.write(str(s["name"]) + '\n') + f.write(str(s["author"]) + '\n') + f.write(str(s["license"])) + url = "/skins/1/" + str(s["id"]) + ".png" + def closure(skinsdir,previewbase,preview,s): + "explanation: python sucks" + def tryget(r): + print('replacing',s["id"]) + if r.status != 200: + print("Error", r.status) + return + @replace(skinsdir,previewbase,path=preview) + def go(f): + shutil.copyfileobj(r,f) + return tryget + + pipeline.append(url,closure(skinsdir,previewbase,preview,s), + "Couldn't get {} because of a".format( + s["id"])) + if not foundOne: + print("No skins updated on this page. Seems we're done?") + #raise SystemExit + addpage(curpage) + while pages > curpage: + curpage = curpage + 1 + addpage(curpage) + print("Skins have been updated!") + diff --git a/updater/update_skins_db.sh b/updater/update_skins_db.sh new file mode 100755 index 0000000..8028b42 --- /dev/null +++ b/updater/update_skins_db.sh @@ -0,0 +1,79 @@ +#!/bin/bash +#### +# Licenced under Attribution-NonCommercial-ShareAlike 4.0 International +# http://creativecommons.org/licenses/by-nc-sa/4.0/ +#### ATTENTION #### +## This script requires that jq and coreutils are installed on your system ## +## In Debian-based distros, open a terminal and run +## sudo apt-get install jq coreutils +################### + +# == Set variables === +# ==================== +NUMPAGES="1" # Number of pages. Default is 1 page +PERPAGE="2000" # Number of items per page. Default is 2000. +JSONURL="http://minetest.fensta.bplaced.net/api/get.json.php?getlist&page=$NUMPAGES&outformat=base64&per_page=$PERPAGE" # The URL to the database +PREVIEWURL="http://minetest.fensta.bplaced.net/skins/1/" # The url to the location of the previews. +curpath="$(dirname $0)" # all path are relative to this script place +temp="$curpath"/tmp # Where the temp folder will be. Default is $PWD/tmp, which means that the tmp folder will be put in the current folder +METADEST="$curpath"/../meta # This is the folder where the meta data will be saved +TEXTUREDEST="$curpath"/../textures # This is the folder where the skins and the previews will be saved + + +# === Make a bunch of folders and download the db === +# =================================================== +if [ -d "$temp" ]; then + rm -r $temp # If the temp dir exists we will remove it and its contents. +fi +mkdir $temp # Make a new temp dir. Redundant? No. We will get rid of it later. + +if [ ! -d "$METADEST" ]; then # Check to see if the meta dir exists, and if not, create it + mkdir $METADEST +fi + +if [ ! -d "$TEXTUREDEST" ]; then # Check to see if the textures dir exists, and if not, create it + mkdir $TEXTUREDEST +fi + +wget $JSONURL -O $temp/rawdb.txt # Download the entire database + + +# === Do the JSON thing === +# ========================= +i="0" # This will be the counter. +while [ "$ID" != "null" ] # Repeat for as long as there is data to process + do + ID=$(cat $temp/rawdb.txt | jq ".skins[$i].id") + + # The next lines are kinda complex. sed is being used to strip the quotes from the variables. I had help... + meta_name=$(echo $(cat $temp/rawdb.txt | jq ".skins[$i].name") | sed -e 's/^"//' -e 's/"$//') + meta_author=$(echo $(cat $temp/rawdb.txt | jq ".skins[$i].author") | sed -e 's/^"//' -e 's/"$//') + meta_license=$(echo $(cat $temp/rawdb.txt | jq ".skins[$i].license") | sed -e 's/^"//' -e 's/"$//') + + if [[ "$ID" != "null" ]]; then # Check to see if ID has a value + echo "#"$ID "name:" $meta_name "author:" $meta_author "license:" $meta_license # Verbosity to show that the script is working. + + echo $meta_name > $METADEST/character_$ID.txt # Save the meta data to files, this line overwrites the data inside the file + echo $meta_author >> $METADEST/character_$ID.txt # Save the meta data to files, this line is added to the file + echo $meta_license >> $METADEST/character_$ID.txt # Save the meta data to files, and this line is added to the file as well. + + + # === Extract and save the image from the JSON file === + # ====================================================== + skin=$(echo $(cat $temp/rawdb.txt | jq ".skins[$i].img") | sed -e 's/^"//' -e 's/"$//') # Strip the quotes from the base64 encoded string + echo $skin | base64 --decode > $TEXTUREDEST"/character_"$ID".png" # Decode the string, and save it as a .png file + + + # === Download a preview image whilst we're at it === + # ==================================================== + wget -nv $PREVIEWURL/$ID".png" -O $TEXTUREDEST"/character_"$ID"_preview.png" # Downloads a preview of the skin that we just saved. + + fi + i=$[$i+1] # Increase the counter by one. + done + +# === Now we'll clean up the mess === +# =================================== +rm -r $temp # Remove the temp dir and its contents. + +exit # Not strictly needed, but i like to use it to wrap things up. |