架构更新

This commit is contained in:
Bigsk 2021-06-20 15:22:39 +08:00
parent 3e02140e97
commit 9c21a1bb26
14 changed files with 360 additions and 257 deletions

View File

@ -1,245 +0,0 @@
/*
Copyright Ghink Network Studio
Website: https://www.ghink.net
*/
package main
import (
"net/http"
"log"
"strings"
"time"
"encoding/json"
"fmt"
"io/ioutil"
"os"
"io"
"crypto/sha256"
"encoding/hex"
"path"
)
var WorkPath,DataPath string
var NodeList,Database,Dataread []map[string]string
var Loger *log.Logger
var ch chan bool
func responseICON(w http.ResponseWriter,r *http.Request) {
w.Header().Set("Location","https://resource.ghink.net/public/img/logo/ghink.png")
w.WriteHeader(301)
}
func responseError(w http.ResponseWriter,r *http.Request,httpCode int,info string,errorCode int) {
w.Header().Set("Content-Type","application/json")
w.WriteHeader(httpCode)
type returnError struct{
HttpCode int `json:"httpCode"`
Info string `json:"info"`
ErrorCode int `json:"errorCode"`
}
returnInfo:=returnError{httpCode,info,errorCode}
b,_:=json.Marshal(returnInfo)
w.Write(b)
}
func responseWiKi(w http.ResponseWriter,r *http.Request) {
w.Header().Set("Content-Type","application/json")
w.WriteHeader(200)
type returnWiki struct{
HttpCode int `json:"httpCode"`
Wiki string `json:"wiki"`
}
returnInfo:=returnWiki{200,"https://gitee.com/ghink/gmclapi/wikis"}
b,_:=json.Marshal(returnInfo)
w.Write(b)
}
func responseOfficial(w http.ResponseWriter,r *http.Request,value []string) {
paths:=""
for _,arg := range value{
paths=paths+"/"+arg
}
if paths=="" || paths=="/"{
responseError(w,r,400,"Please offer more params",10005)
}else{
switcher:=true
for _,arg := range Database{
if arg["path"]==paths{
address:=determineNode(arg["hash"],path.Base(arg["path"]))
w.Header().Set("Location",address)
w.WriteHeader(302)
switcher=false
break
}
}
if switcher==true{
responseError(w,r,404,"Cannot find file",10003)
}
}
}
func responseFile(w http.ResponseWriter,r *http.Request,value []string) {
fileName:=value[0]
path:=DataPath
for _,arg := range value[1:]{
path=path+"/"+arg
}
if path==DataPath{
responseError(w,r,403,"Directory listing is not allowed",10002)
}else{
b,err:=ioutil.ReadFile(path)
if err != nil {
if find := strings.Contains(err.Error(),"The system cannot find"); find{
responseError(w,r,404,"Cannot find file",10003)
}else if s,_:=os.Stat(path); s.IsDir(){
responseError(w,r,403,"Directory listing is not allowed",10002)
}else{
responseError(w,r,500,"The file exists but cannot be read",10004)
logIn("[Page Thread]","Read the file ",path," failed in data dir! Error info: ",err.Error())
}
}else{
w.Header().Set("Content-Disposition","attachment;filename="+fileName)
w.Header().Set("Content-Type","application/octet-stream")
w.Write(b)
}
}
}
func pageService(w http.ResponseWriter,r *http.Request) {
ts := time.Now().Unix()
logIn("[Page Thread]","Got requests from ",r.RemoteAddr,", address is ",r.URL.RequestURI())
w.Header().Set("Server","GMCLAPI (Alpha 1.0.0)")
w.Header().Set("Engine","Panda (Alpha 1.0.0)")
w.Header().Set("Stamp",fmt.Sprint(ts))
addr:=r.URL.RequestURI()
value:=strings.Split(addr,"/")
pathType:=value[1]
value=value[2:]
if addr=="/"{
responseWiKi(w,r)
}else{
switch pathType {
case "favicon.ico":
responseICON(w,r)
case "file":
responseFile(w,r,value)
case "official":
responseOfficial(w,r,value)
/*
case "bmclapi":
responseBMCLAPI(w,r,value)
case "default":
responseDefault(w,r,value)
case "gmclapi":
responseDefault(w,r,value)
*/
default:
responseError(w,r,405,"Unspported choice for requests loading",10001)
}
}
}
func databaseService() {
for {
defer recover()
file,err:=os.OpenFile(WorkPath+"/database.db",os.O_RDONLY,0600)
defer file.Close()
if err != nil {
panic(err)
}
bytes,err:=ioutil.ReadAll(file)
if err != nil {
panic(err)
}
err=json.Unmarshal(bytes,&Database)
if err != nil {
panic(err)
}
time.Sleep(time.Duration(10)*time.Second)
}
}
func nodeService() {
}
func determineNode(hash string,fileName string) string {
return "http://127.0.0.1:8000/file/"+fileName+"/"+hash
}
func syncService() {
for {
}
}
func hashFile(path string) string {
file,err:=os.Open(path)
defer file.Close()
if err != nil {
logIn("[Hash Thread]","Failed to calc hash of the file ",path,"! Error info: ",err.Error())
}
hash:=sha256.New()
if _,err := io.Copy(hash,file); err != nil {
logIn("[Hash Thread]","Failed to calc hash of the file ",path,"! Error info: ",err.Error())
}
sum:=hash.Sum(nil)
return hex.EncodeToString(sum)
}
func logIn(text ...string) {
result:=""
for _,arg := range text {
result+=arg
}
log.Print(result)
Loger.Print(result)
}
func init() {
WorkPath="gmclapi/"
DataPath=WorkPath+"data/"
err:=os.MkdirAll(WorkPath, os.ModePerm)
if err != nil {
panic(err)
}
err=os.MkdirAll(DataPath, os.ModePerm)
if err != nil {
panic(err)
}
if _,err:=os.Stat(WorkPath+"database.db"); os.IsNotExist(err){
database,err:=os.OpenFile(WorkPath+"database.db", os.O_WRONLY|os.O_CREATE, 0666)
database.Write([]byte("[]"))
if err != nil {
panic(err)
}
defer database.Close()
}
file,err:=os.OpenFile(WorkPath+"/database.db", os.O_RDONLY,0600)
defer file.Close()
if err != nil {
panic(err)
}
bytes,err:=ioutil.ReadAll(file)
if err != nil {
panic(err)
}
err=json.Unmarshal(bytes,&Database)
if err != nil {
panic(err)
}
}
func main() {
logFile,err:=os.OpenFile(WorkPath+"logs.log", os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666)
if err != nil {
panic(err)
}
defer logFile.Close()
Loger=log.New(logFile,"",log.LstdFlags|log.Lshortfile)
http.HandleFunc("/", pageService)
go http.ListenAndServe("127.0.0.1:8000", nil)
go syncService()
go databaseService()
/*
go nodeService()
go consoleService()
*/
for {
}
}
/*
10001 Unspported choice for requests loading
10002 Directory listing is not allowed
10003 Cannot find file
10004 The file exists but cannot be read
10005 Please offer more params
*/

View File

@ -0,0 +1 @@
package database

1
master/src/main/bugs.md Normal file
View File

@ -0,0 +1 @@
Log不能被正常记录

80
master/src/main/main.go Normal file
View File

@ -0,0 +1,80 @@
/*
Copyright Ghink Network Studio
Website: https://www.ghink.net
*/
package main
import (
"net/http"
"time"
"encoding/json"
"io/ioutil"
"os"
"io"
"crypto/sha256"
"encoding/hex"
"vars"
"page/service"
"tools/loger"
)
func databaseService() {
for {
defer recover()
file,err:=os.OpenFile(vars.WorkPath+"/database.db",os.O_RDONLY,0600)
defer file.Close()
if err != nil {
continue
}
bytes,err:=ioutil.ReadAll(file)
if err != nil {
continue
}
err=json.Unmarshal(bytes,&vars.Database)
if err != nil {
continue
}
time.Sleep(time.Duration(10)*time.Second)
}
}
func nodeService() {
}
func syncService() {
for {
}
}
func hashFile(path string) string {
file,err:=os.Open(path)
defer file.Close()
if err != nil {
loger.LogIn("[Hash Thread]","Failed to calc hash of the file ",path,"! Error info: ",err.Error())
}
hash:=sha256.New()
if _,err := io.Copy(hash,file); err != nil {
loger.LogIn("[Hash Thread]","Failed to calc hash of the file ",path,"! Error info: ",err.Error())
}
sum:=hash.Sum(nil)
return hex.EncodeToString(sum)
}
func main() {
http.HandleFunc("/", service.PageService)
go http.ListenAndServe(":8000", nil)
go syncService()
go databaseService()
/*
go nodeService()
go consoleService()
*/
for {
}
}
/*
10001 Unspported choice for requests loading
10002 Directory listing is not allowed
10003 Cannot find file
10004 The file exists but cannot be read
10005 Please offer more params
*/

View File

@ -24,8 +24,11 @@ def log(info):
with open(workPath+"logs.log","a+") as fb:
fb.write(info+"\n")
def database():
threadLock[1]+=1
global db
while True:
if threadLock[1]<threadLock[0]:
break
threadLock[1]+=1
with open(workPath+"database.db","r") as fb:
db=json.loads(fb.read())
while True:
@ -66,9 +69,16 @@ def syncMain():
except:
pass
def syncJson():
while True:
if threadLock[1]<threadLock[0]:
break
threadLock[1]+=1
synced=[]
def syncThread(rec):
global db
while True:
if threadLock[1]<threadLock[0]:
break
threadLock[1]+=1
with open(dataPath+rec["hash"],"r") as fb:
fall=re.findall('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+',fb.read())
@ -109,16 +119,23 @@ def syncJson():
threading.Thread(target=syncThread,args=(rec,)).start()
if i==0:
break
log("Synchronizing for json list finished")
threadLock[1]-=1
def syncForge(entrance="https://files.minecraftforge.net/net/minecraftforge/forge/"):
global db
while True:
if threadLock[1]<threadLock[0]:
break
threadLock[1]+=1
crawed=[]
def syncThread(entrance):
global db
while True:
if threadLock[1]<threadLock[0]:
break
threadLock[1]+=1
log("Crawling the page {}".format(entrance))
page=requests.get(entrance,headers=userAgent,proxies=proxies).text
soup=bs(page,features="html5lib")
soup=bs(page,features="html.parser")
pageurls=soup.find_all("a",href=True)
for obj in pageurls:
link=obj.get("href")
@ -173,18 +190,21 @@ def syncForge(entrance="https://files.minecraftforge.net/net/minecraftforge/forg
time.sleep(10)
threadLock[1]-=1
syncThread(entrance)
def monitor():
threadLock[1]-=1
def serviceManager():
while True:
log(str(threadLock))
time.sleep(1)
def main():
threading.Thread(target=database).start()
#threading.Thread(target=monitor).start()
time.sleep(1)
while True:
time.sleep(60*60*24)
syncMain()
threading.Thread(target=syncJson()).start()
threading.Thread(target=syncForge()).start()
time.sleep(60*60*24)
def main():
threadLock[1]+=1
threading.Thread(target=database).start()
time.sleep(1)
threading.Thread(target=serviceManager).start()
while True:
command=input()
if command=="thread":
log("There's "+str(threadLock[1])+" threads are running, totally limited set as "+str(threadLock[0]))
if __name__=="__main__":
main()

View File

View File

@ -0,0 +1,19 @@
package errors
import (
"net/http"
"encoding/json"
)
func ResponseError(w http.ResponseWriter,r *http.Request,httpCode int,info string,errorCode int) {
w.Header().Set("Content-Type","application/json")
w.WriteHeader(httpCode)
type returnError struct{
HttpCode int `json:"httpCode"`
Info string `json:"info"`
ErrorCode int `json:"errorCode"`
}
returnInfo:=returnError{httpCode,info,errorCode}
b,_:=json.Marshal(returnInfo)
w.Write(b)
}

View File

@ -0,0 +1,42 @@
package file
import (
"net/http"
"strings"
"io/ioutil"
"os"
"page/response/errors"
"tools/loger"
"vars"
)
func ResponseFile(w http.ResponseWriter,r *http.Request,value []string) {
test:=""
for _,arg := range value{
test=test+"/"+arg
}
if test==""{
errors.ResponseError(w,r,403,"Directory listing is not allowed",10002)
}else{
fileName:=value[0]
path:=vars.DataPath
for _,arg := range value[1:]{
path=path+"/"+arg
}
b,err:=ioutil.ReadFile(path)
if err != nil {
if find := strings.Contains(err.Error(),"The system cannot find"); find{
errors.ResponseError(w,r,404,"Cannot find file",10003)
}else if s,_:=os.Stat(path); s.IsDir(){
errors.ResponseError(w,r,403,"Directory listing is not allowed",10002)
}else{
errors.ResponseError(w,r,500,"The file exists but cannot be read",10004)
loger.LogIn("[Page Thread]","Read the file ",path," failed in data dir! Error info: ",err.Error())
}
}else{
w.Header().Set("Content-Disposition","attachment;filename="+fileName)
w.Header().Set("Content-Type","application/octet-stream")
w.Write(b)
}
}
}

View File

@ -0,0 +1,10 @@
package icon
import (
"net/http"
)
func ResponseICON(w http.ResponseWriter,r *http.Request) {
w.Header().Set("Location","https://resource.ghink.net/public/img/logo/ghink.png")
w.WriteHeader(301)
}

View File

@ -0,0 +1,32 @@
package official
import (
"net/http"
"page/response/errors"
"vars"
"path"
)
func ResponseOfficial(w http.ResponseWriter,r *http.Request,value []string) {
paths:=""
for _,arg := range value{
paths=paths+"/"+arg
}
if paths=="" || paths=="/"{
errors.ResponseError(w,r,400,"Please offer more params",10005)
}else{
switcher:=true
for _,arg := range vars.Database{
if arg["path"]==paths{
address:=vars.DetermineNode(arg["hash"],path.Base(arg["path"]))
w.Header().Set("Location",address)
w.WriteHeader(302)
switcher=false
break
}
}
if switcher==true{
errors.ResponseError(w,r,404,"Cannot find file",10003)
}
}
}

View File

@ -0,0 +1,18 @@
package wiki
import (
"net/http"
"encoding/json"
)
func ResponseWiKi(w http.ResponseWriter,r *http.Request) {
w.Header().Set("Content-Type","application/json")
w.WriteHeader(200)
type returnWiki struct{
HttpCode int `json:"httpCode"`
Wiki string `json:"wiki"`
}
returnInfo:=returnWiki{200,"https://gitee.com/ghink/gmclapi/wikis"}
b,_:=json.Marshal(returnInfo)
w.Write(b)
}

View File

@ -0,0 +1,48 @@
package service
import (
"net/http"
"strings"
"time"
"fmt"
"tools/loger"
"page/response/icon"
"page/response/wiki"
"page/response/file"
"page/response/official"
"page/response/errors"
)
func PageService(w http.ResponseWriter,r *http.Request) {
ts := time.Now().Unix()
loger.LogIn("[Page Thread]","Got requests from ",r.RemoteAddr,", address is ",r.URL.RequestURI())
w.Header().Set("Server","GMCLAPI (Alpha 1.0.0)")
w.Header().Set("Engine","Panda (Alpha 1.0.0)")
w.Header().Set("Stamp",fmt.Sprint(ts))
addr:=r.URL.RequestURI()
value:=strings.Split(addr,"/")
pathType:=value[1]
value=value[2:]
if addr=="/"{
wiki.ResponseWiKi(w,r)
}else{
switch pathType {
case "favicon.ico":
icon.ResponseICON(w,r)
case "file":
file.ResponseFile(w,r,value)
case "official":
official.ResponseOfficial(w,r,value)
/*
case "bmclapi":
responseBMCLAPI(w,r,value)
case "default":
responseDefault(w,r,value)
case "gmclapi":
responseDefault(w,r,value)
*/
default:
errors.ResponseError(w,r,405,"Unspported choice for requests loading",10001)
}
}
}

View File

@ -0,0 +1,27 @@
package loger
import (
"fmt"
"log"
"os"
"vars"
)
var Loger *log.Logger
func init() {
logFile,err:=os.OpenFile(vars.WorkPath+"logs.log", os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666)
if err != nil {
panic(err)
}
defer logFile.Close()
Loger=log.New(logFile,"",log.LstdFlags|log.Lshortfile)
}
func LogIn(text ...interface{}) {
result:=""
for _,arg := range text {
result+=fmt.Sprint(arg)
}
log.Print(result)
Loger.Print(result)
}

50
master/src/vars/vars.go Normal file
View File

@ -0,0 +1,50 @@
package vars
import(
"os"
"io/ioutil"
"encoding/json"
)
var WorkPath,DataPath string
var NodeList,Database,Dataread []map[string]string
func init() {
WorkPath="gmclapi/"
DataPath=WorkPath+"data/"
err:=os.MkdirAll(WorkPath, os.ModePerm)
if err != nil {
panic(err)
}
err=os.MkdirAll(DataPath, os.ModePerm)
if err != nil {
panic(err)
}
if _,err:=os.Stat(WorkPath+"database.db"); os.IsNotExist(err){
database,err:=os.OpenFile(WorkPath+"database.db", os.O_WRONLY|os.O_CREATE, 0666)
database.Write([]byte("[]"))
if err != nil {
panic(err)
}
defer database.Close()
}
for {
file,err:=os.OpenFile(WorkPath+"/database.db", os.O_RDONLY,0600)
defer file.Close()
if err != nil {
continue
}
bytes,err:=ioutil.ReadAll(file)
if err != nil {
continue
}
err=json.Unmarshal(bytes,&Database)
if err != nil {
continue
}
break
}
}
func DetermineNode(hash string,fileName string) string {
return "http://1.cn.server.ghink.net:8000/file/"+fileName+"/"+hash
}