From 3e02140e97ba40ced98d68dd5bdeff205ab67d00 Mon Sep 17 00:00:00 2001 From: Bigsk Date: Mon, 14 Jun 2021 15:35:53 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=AE=98=E6=96=B9=E5=93=8D?= =?UTF-8?q?=E5=BA=94=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- master/{main => }/main.go | 41 ++++++-- master/sync.py | 190 ++++++++++++++++++++++++++++++++++++++ master/sync/sync.py | 97 ------------------- 3 files changed, 225 insertions(+), 103 deletions(-) rename master/{main => }/main.go (84%) create mode 100644 master/sync.py delete mode 100644 master/sync/sync.py diff --git a/master/main/main.go b/master/main.go similarity index 84% rename from master/main/main.go rename to master/main.go index b8d90f0..76ea81c 100644 --- a/master/main/main.go +++ b/master/main.go @@ -17,6 +17,7 @@ import ( "io" "crypto/sha256" "encoding/hex" + "path" ) var WorkPath,DataPath string @@ -51,12 +52,36 @@ func responseWiKi(w http.ResponseWriter,r *http.Request) { b,_:=json.Marshal(returnInfo) w.Write(b) } -func responseFile(w http.ResponseWriter,r *http.Request,value []string) { - path:=DataPath +func responseOfficial(w http.ResponseWriter,r *http.Request,value []string) { + paths:="" for _,arg := range value{ + paths=paths+"/"+arg + } + if paths=="" || paths=="/"{ + responseError(w,r,400,"Please offer more params",10005) + }else{ + switcher:=true + for _,arg := range Database{ + if arg["path"]==paths{ + address:=determineNode(arg["hash"],path.Base(arg["path"])) + w.Header().Set("Location",address) + w.WriteHeader(302) + switcher=false + break + } + } + if switcher==true{ + responseError(w,r,404,"Cannot find file",10003) + } + } +} +func responseFile(w http.ResponseWriter,r *http.Request,value []string) { + fileName:=value[0] + path:=DataPath + for _,arg := range value[1:]{ path=path+"/"+arg } - if path==""{ + if path==DataPath{ responseError(w,r,403,"Directory listing is not allowed",10002) }else{ b,err:=ioutil.ReadFile(path) @@ -70,6 +95,8 @@ func responseFile(w http.ResponseWriter,r *http.Request,value []string) { logIn("[Page Thread]","Read the file ",path," failed in data dir! Error info: ",err.Error()) } }else{ + w.Header().Set("Content-Disposition","attachment;filename="+fileName) + w.Header().Set("Content-Type","application/octet-stream") w.Write(b) } } @@ -92,9 +119,9 @@ func pageService(w http.ResponseWriter,r *http.Request) { responseICON(w,r) case "file": responseFile(w,r,value) - /* case "official": responseOfficial(w,r,value) + /* case "bmclapi": responseBMCLAPI(w,r,value) case "default": @@ -109,6 +136,7 @@ func pageService(w http.ResponseWriter,r *http.Request) { } func databaseService() { for { + defer recover() file,err:=os.OpenFile(WorkPath+"/database.db",os.O_RDONLY,0600) defer file.Close() if err != nil { @@ -128,8 +156,8 @@ func databaseService() { func nodeService() { } -func determineNode(file string) string { - return "http://127.0.0.1/file"+file +func determineNode(hash string,fileName string) string { + return "http://127.0.0.1:8000/file/"+fileName+"/"+hash } func syncService() { for { @@ -213,4 +241,5 @@ func main() { 10002 Directory listing is not allowed 10003 Cannot find file 10004 The file exists but cannot be read +10005 Please offer more params */ \ No newline at end of file diff --git a/master/sync.py b/master/sync.py new file mode 100644 index 0000000..45c5fa9 --- /dev/null +++ b/master/sync.py @@ -0,0 +1,190 @@ +''' +Copyright Ghink Network Studio +Website: https://www.ghink.net +''' +from io import BufferedRandom +import urllib +import requests,os,time,json,threading,hashlib,re +from urllib import parse +from bs4 import BeautifulSoup as bs + +db=[] +workPath="gmclapi/" +dataPath=workPath+"data/" +userAgent={'User-Agent':'GMCLAPI/0.0.1'} +proxies = { + "http": "http://127.0.0.1:4780" +} +entrance=["http://launchermeta.mojang.com/mc/game/version_manifest.json","http://dl.liteloader.com/versions/versions.json"] +threadLock=[200,0] + +def log(info): + info="{}{}".format(time.strftime("%Y/%m/%d %H:%M:%S [Sync Thread]", time.localtime()),info) + print(info) + with open(workPath+"logs.log","a+") as fb: + fb.write(info+"\n") +def database(): + threadLock[1]+=1 + global db + with open(workPath+"database.db","r") as fb: + db=json.loads(fb.read()) + while True: + with open(workPath+"database.db","r") as fb: + if json.loads(fb.read())==db: + continue + with open(workPath+"database.db","w+") as fb: + fb.write(json.dumps(db)) +def syncMain(): + global db + for obj in entrance: + while True: + try: + origin=requests.get(obj,headers=userAgent,proxies=proxies).content + md5=hashlib.md5() + md5.update(origin) + hash=md5.hexdigest() + switch=True + for h in db: + if h["hash"]==hash: + switch=False + break + if switch: + log("Synchronizing the file {} from {}".format(parse.urlparse(obj).path,obj)) + i=0 + for h in db: + if h["path"]==parse.urlparse(obj).path: + del switch[i] + i+=1 + with open(dataPath+hash,"wb") as fb: + fb.write(origin) + db.append({ + "hash":hash, + "source":obj, + "path":parse.urlparse(obj).path + }) + break + except: + pass +def syncJson(): + synced=[] + def syncThread(rec): + global db + threadLock[1]+=1 + with open(dataPath+rec["hash"],"r") as fb: + fall=re.findall('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+',fb.read()) + for obj in fall: + if parse.urlparse(obj).path=="/": + continue + while True: + try: + switch=True + for r in db: + if r["path"]==parse.urlparse(obj).path: + switch=False + break + if switch: + log("Synchronizing the file {} from {}".format(parse.urlparse(obj).path,obj)) + origin=requests.get(obj,headers=userAgent,proxies=proxies).content + md5=hashlib.md5() + md5.update(origin) + hash=md5.hexdigest() + with open(dataPath+hash,"wb") as fb: + fb.write(origin) + db.append({ + "hash":hash, + "source":obj, + "path":parse.urlparse(obj).path + }) + break + except: + pass + threadLock[1]-=1 + while True: + i=0 + for rec in db: + if ".json" in rec["path"] and rec["path"] not in synced: + i+=1 + if threadLock[0]>threadLock[1]: + synced.append(rec["path"]) + threading.Thread(target=syncThread,args=(rec,)).start() + if i==0: + break + log("Synchronizing for json list finished") +def syncForge(entrance="https://files.minecraftforge.net/net/minecraftforge/forge/"): + global db + crawed=[] + def syncThread(entrance): + global db + threadLock[1]+=1 + log("Crawling the page {}".format(entrance)) + page=requests.get(entrance,headers=userAgent,proxies=proxies).text + soup=bs(page,features="html5lib") + pageurls=soup.find_all("a",href=True) + for obj in pageurls: + link=obj.get("href") + if ".html" in link and "http" not in link and link not in crawed: + crawed.append(link) + threading.Thread(target=syncThread,args=(parse.urljoin(entrance,link),)).start() + fall=re.findall('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+',page) + for obj in fall: + switch=True + for h in db: + if h["path"]==parse.urlparse(obj).path or h["path"]==parse.urlparse(obj[48:]).path: + switch=False + break + if switch: + if ".jar" in obj or ".zip" in obj: + log("Synchronizing the file {} from {}".format(parse.urlparse(obj[48:]).path,obj[48:])) + while True: + try: + origin=requests.get(obj[48:],headers=userAgent,proxies=proxies).content + md5=hashlib.md5() + md5.update(origin) + hash=md5.hexdigest() + with open(dataPath+hash,"wb") as fb: + fb.write(origin) + db.append({ + "hash":hash, + "source":obj[48:], + "path":parse.urlparse(obj[48:]).path + }) + break + except: + log("Synchronizing the file {} from {} failed!Retrying...".format(parse.urlparse(obj[48:]).path,obj[48:])) + time.sleep(10) + elif ".txt" in link: + log("Synchronizing the file {} from {}".format(parse.urlparse(obj).path,obj)) + while True: + try: + origin=requests.get(obj,headers=userAgent,proxies=proxies).content + md5=hashlib.md5() + md5.update(origin) + hash=md5.hexdigest() + with open(dataPath+hash,"wb") as fb: + fb.write(origin) + db.append({ + "hash":hash, + "source":obj, + "path":parse.urlparse(obj).path + }) + break + except: + log("Synchronizing the file {} from {} failed!Retrying...".format(parse.urlparse(obj).path,obj)) + time.sleep(10) + threadLock[1]-=1 + syncThread(entrance) +def monitor(): + while True: + log(str(threadLock)) + time.sleep(1) +def main(): + threading.Thread(target=database).start() + #threading.Thread(target=monitor).start() + time.sleep(1) + while True: + time.sleep(60*60*24) + syncMain() + threading.Thread(target=syncJson()).start() + threading.Thread(target=syncForge()).start() +if __name__=="__main__": + main() \ No newline at end of file diff --git a/master/sync/sync.py b/master/sync/sync.py deleted file mode 100644 index 212fad3..0000000 --- a/master/sync/sync.py +++ /dev/null @@ -1,97 +0,0 @@ -''' -Copyright Ghink Network Studio -Website: https://www.ghink.net -''' -import requests,os,time,json,threading,hashlib,re -from urllib.parse import urlparse - -db=[] -workPath="gmclapi/" -dataPath=workPath+"data/" -userAgent={'User-Agent':'GMCLAPI/0.0.1'} -proxies = { - "http": "http://127.0.0.1:4780" -} -entrance=["http://launchermeta.mojang.com/mc/game/version_manifest.json"] - -def log(info): - info="{}{}".format(time.strftime("%Y/%m/%d %H:%M:%S [Sync Thread]", time.localtime()),info) - print(info) - with open(workPath+"logs.log","a+") as fb: - fb.write(info) - fb.write("\r\n") -def database(): - global db - with open(workPath+"database.db","r") as fb: - db=json.loads(fb.read()) - while True: - with open(workPath+"database.db","r") as fb: - if json.loads(fb.read())==db: - continue - with open(workPath+"database.db","w+") as fb: - fb.write(json.dumps(db)) -def syncMain(): - global entrance - global userAgent - global proxies - for obj in entrance: - origin=requests.get(obj,headers=userAgent,proxies=proxies).content - md5=hashlib.md5() - md5.update(origin) - hash=md5.hexdigest() - switch=True - for h in db: - if h["hash"]==hash: - switch=False - break - if switch: - i=0 - for h in db: - if h["path"]==urlparse(obj).path: - del switch[i] - i+=1 - with open(dataPath+hash,"wb") as fb: - fb.write(origin) - db.append({ - "hash":hash, - "source":obj, - "path":urlparse(obj).path - }) -def syncMinecraft(): - global userAgent - global proxies - for h in db: - if h["path"]=="/mc/game/version_manifest.json": - hash=h["hash"] - break - with open(dataPath+hash,"r") as fb: - fall=fb.read() - result=re.findall('(https?|ftp|file)|([-A-Za-z0-9+&@#/%?=~_|!:,.;]+[-A-Za-z0-9+&@#/%=~_|])',fall) - print(result) - res=[] - for i in result: - res.append("{}{}".format(i[0]+i[1])) - for obj in res: - switch=True - for h in db: - if h["source"]==obj: - switch=False - if switch: - origin=requests.get(obj,headers=userAgent,proxies=proxies).content - md5=hashlib.md5() - md5.update(origin) - hash=md5.hexdigest() - with open(dataPath+hash,"wb") as fb: - fb.write(origin) - db.append({ - "hash":hash, - "source":obj, - "path":urlparse(obj).path - }) -def main(): - threading.Thread(target=database).start() - syncMain() -#main() -threading.Thread(target=database).start() -time.sleep(1) -syncMinecraft() \ No newline at end of file