diff options
Diffstat (limited to 'update_from_db.py')
| -rwxr-xr-x | update_from_db.py | 213 |
1 files changed, 166 insertions, 47 deletions
diff --git a/update_from_db.py b/update_from_db.py index be6930c..159acf7 100755 --- a/update_from_db.py +++ b/update_from_db.py @@ -1,61 +1,180 @@ #!/usr/bin/python3 -from http.client import HTTPConnection +from http.client import HTTPConnection,HTTPException,BadStatusLine,_CS_IDLE import json import base64 +from contextlib import closing +import sys,os,shutil,time + +def die(message,code=23): + print(message,file=sys.stderr) + raise SystemExit(code) server = "minetest.fensta.bplaced.net" skinsdir = "u_skins/textures/" metadir = "u_skins/meta/" -i = 1 -pages = 1 +curskin = 0 +curpage = 1 +pages = None -c = HTTPConnection(server) -def addpage(page): - global i, pages - print("Page: " + str(page)) - r = 0 - try: - c.request("GET", "/api/get.json.php?getlist&page=" + str(page) + "&outformat=base64") - r = c.getresponse() - except Exception: - if r != 0: - if r.status != 200: - print("Error", r.status) - exit(r.status) - return - - data = r.read().decode() - l = json.loads(data) - if not l["success"]: - print("Success != True") - exit(1) - r = 0 - pages = int(l["pages"]) - for s in l["skins"]: - f = open(skinsdir + "character_" + str(i) + ".png", "wb") - f.write(base64.b64decode(bytes(s["img"], 'utf-8'))) - f.close() - f = open(metadir + "character_" + str(i) + ".txt", "w") - f.write(str(s["name"]) + '\n') - f.write(str(s["author"]) + '\n') - f.write(str(s["license"])) - f.close() +def replace(location,base,encoding=None,path=None): + if path is None: + path = os.path.join(location,base) + mode = "wt" if encoding else "wb" + # an unpredictable temp name only needed for a+rwxt directories + tmp = os.path.join(location,'.'+base+'-tmp') + def deco(handle): + with open(tmp,mode,encoding=encoding) as out: + handle(out) + os.rename(tmp,path) + return deco + +def maybeReplace(location,base,encoding=None): + def deco(handle): + path = os.path.join(location,base) + if os.path.exists(path): return + return replace(location,base,encoding=encoding,path=path)(handle) + return deco + +class Penguin: + "idk" + def __init__(self, url, recv, diemessage): + self.url = url + self.recv = recv + self.diemessage = diemessage + +class Pipeline(list): + "Gawd why am I being so elaborate?" + def __init__(self, threshold=10): + "threshold is how many requests in parallel to pipeline" + self.threshold = threshold + self.sent = True + def __enter__(self): + self.reopen() + return self + def __exit__(self,typ,exn,trace): + self.send() + self.drain() + def reopen(self): + self.c = HTTPConnection(server) + self.send() + def append(self,url,recv,diemessage): + self.sent = False + super().append(Penguin(url,recv,diemessage)) + if len(self) > self.threshold: + self.send() + self.drain() + def trydrain(self): + for penguin in self: + print('drain',penguin.url) + try: + penguin.response.begin() + penguin.recv(penguin.response) + except BadStatusLine as e: + print('derped requesting',penguin.url) + return False + except HTTPException as e: + die(penguin.diemessage+' '+repr(e)+' (url='+penguin.url+')') + self.clear() + return True + def drain(self): + print('draining pipeline...',len(self)) + assert self.sent, "Can't drain without sending the requests!" + self.sent = False + while self.trydrain() is not True: + self.c.close() + print('drain failed, trying again') + time.sleep(1) + self.reopen() + def trysend(self): + for penguin in pipeline: + print('fill',penguin.url) + try: + self.c.request("GET", penguin.url) + self.c._HTTPConnection__state = _CS_IDLE + penguin.response = self.c.response_class(self.c.sock, + method="GET") + # begin LATER so we can send multiple requests w/out response headers + except BadStatusLine: + return False + except HTTPException as e: + die(diemessage+' because of a '+repr(e)) + return True + def send(self): + if self.sent: return + print('filling pipeline...',len(self)) + while self.trysend() is not True: + self.c.close() + print('derped resending') + time.sleep(1) + self.reopen() + self.sent = True + +with Pipeline() as pipeline: + # two connections is okay, right? one for json, one for preview images + c = HTTPConnection(server) + def addpage(page): + global curskin, pages + print("Page: " + str(page)) + r = 0 try: - c.request("GET", "/skins/1/" + str(s["id"]) + ".png") + c.request("GET", "/api/get.json.php?getlist&page=" + str(page) + "&outformat=base64") r = c.getresponse() except Exception: if r != 0: if r.status != 200: - print("Error", r.status) - continue + die("Error", r.status) + return - data = r.read() - f = open(skinsdir + "character_" + str(i) + "_preview.png", "wb") - f.write(data) - f.close() - i = i + 1 -addpage(1) -if pages > 1: - for p in range(pages-1): - addpage(p+2) -print("Skins have been updated!") + data = r.read().decode() + l = json.loads(data) + if not l["success"]: + die("Success != True") + r = 0 + pages = int(l["pages"]) + foundOne = False + for s in l["skins"]: + # make sure to increment this, even if the preview exists! + curskin = curskin + 1 + previewbase = "character_" + str(curskin) + "_preview.png" + preview = os.path.join(skinsdir, previewbase) + if os.path.exists(preview): + print('skin',curskin,'already retrieved') + continue + print('updating skin',curskin,'id',s["id"]) + foundOne = True + @maybeReplace(skinsdir, "character_" + str(curskin) + ".png") + def go(f): + f.write(base64.b64decode(bytes(s["img"], 'utf-8'))) + f.close() + + @maybeReplace(metadir, "character_" + str(curskin) + ".txt", + encoding='utf-8') + def go(f): + f.write(str(s["name"]) + '\n') + f.write(str(s["author"]) + '\n') + f.write(str(s["license"])) + url = "/skins/1/" + str(s["id"]) + ".png" + def closure(skinsdir,previewbase,preview,s): + "explanation: python sucks" + def tryget(r): + print('replacing',s["id"]) + if r.status != 200: + print("Error", r.status) + return + @replace(skinsdir,previewbase,path=preview) + def go(f): + shutil.copyfileobj(r,f) + return tryget + + pipeline.append(url,closure(skinsdir,previewbase,preview,s), + "Couldn't get {} because of a".format( + s["id"])) + if not foundOne: + print("No skins updated on this page. Seems we're done?") + #raise SystemExit + addpage(curpage) + while pages > curpage: + curpage = curpage + 1 + addpage(curpage) + print("Skins have been updated!") + |