架构更新

master/main.go

@@ -1,245 +0,0 @@
-/*
-Copyright Ghink Network Studio
-Website: https://www.ghink.net
-*/
-
-package main
- 
-import (
-   "net/http"
-   "log"
-   "strings"
-   "time"
-   "encoding/json"
-   "fmt"
-   "io/ioutil"
-   "os"
-   "io"
-   "crypto/sha256"
-   "encoding/hex"
-   "path"
-)
-
-var WorkPath,DataPath string
-var NodeList,Database,Dataread []map[string]string
-var Loger *log.Logger
-var ch chan bool
-
-func responseICON(w http.ResponseWriter,r *http.Request)  {
-   w.Header().Set("Location","https://resource.ghink.net/public/img/logo/ghink.png")
-   w.WriteHeader(301)
-}
-func responseError(w http.ResponseWriter,r *http.Request,httpCode int,info string,errorCode int)  {
-   w.Header().Set("Content-Type","application/json")
-   w.WriteHeader(httpCode)
-   type returnError struct{
-      HttpCode int `json:"httpCode"`
-      Info string `json:"info"`
-      ErrorCode int `json:"errorCode"`
-   }
-   returnInfo:=returnError{httpCode,info,errorCode}
-   b,_:=json.Marshal(returnInfo)
-   w.Write(b)
-}
-func responseWiKi(w http.ResponseWriter,r *http.Request)  {
-   w.Header().Set("Content-Type","application/json")
-   w.WriteHeader(200)
-   type returnWiki struct{
-      HttpCode int `json:"httpCode"`
-      Wiki string `json:"wiki"`
-   }
-   returnInfo:=returnWiki{200,"https://gitee.com/ghink/gmclapi/wikis"}
-   b,_:=json.Marshal(returnInfo)
-   w.Write(b)
-}
-func responseOfficial(w http.ResponseWriter,r *http.Request,value []string)  {
-   paths:=""
-   for _,arg := range value{
-      paths=paths+"/"+arg
-   }
-   if paths=="" || paths=="/"{
-      responseError(w,r,400,"Please offer more params",10005)
-   }else{
-      switcher:=true
-      for _,arg := range Database{
-         if arg["path"]==paths{
-            address:=determineNode(arg["hash"],path.Base(arg["path"]))
-            w.Header().Set("Location",address)
-            w.WriteHeader(302)
-            switcher=false
-            break
-         }
-      }
-      if switcher==true{
-         responseError(w,r,404,"Cannot find file",10003)
-      }
-   }
-}
-func responseFile(w http.ResponseWriter,r *http.Request,value []string)  {
-   fileName:=value[0]
-   path:=DataPath
-   for _,arg := range value[1:]{
-      path=path+"/"+arg
-   }
-   if path==DataPath{
-      responseError(w,r,403,"Directory listing is not allowed",10002)
-   }else{
-      b,err:=ioutil.ReadFile(path)
-      if err != nil {
-         if find := strings.Contains(err.Error(),"The system cannot find"); find{
-            responseError(w,r,404,"Cannot find file",10003)
-         }else if s,_:=os.Stat(path); s.IsDir(){
-            responseError(w,r,403,"Directory listing is not allowed",10002)
-         }else{
-            responseError(w,r,500,"The file exists but cannot be read",10004)
-            logIn("[Page Thread]","Read the file ",path," failed in data dir! Error info: ",err.Error())
-         }
-      }else{
-         w.Header().Set("Content-Disposition","attachment;filename="+fileName)
-         w.Header().Set("Content-Type","application/octet-stream")
-         w.Write(b)
-      }
-   }
-}
-func pageService(w http.ResponseWriter,r *http.Request) {
-   ts := time.Now().Unix()
-   logIn("[Page Thread]","Got requests from ",r.RemoteAddr,", address is ",r.URL.RequestURI())
-   w.Header().Set("Server","GMCLAPI (Alpha 1.0.0)")
-   w.Header().Set("Engine","Panda (Alpha 1.0.0)")
-   w.Header().Set("Stamp",fmt.Sprint(ts))
-   addr:=r.URL.RequestURI()
-   value:=strings.Split(addr,"/")
-   pathType:=value[1]
-   value=value[2:]
-   if addr=="/"{
-      responseWiKi(w,r)
-   }else{
-      switch pathType {
-      case "favicon.ico":
-         responseICON(w,r)
-      case "file":
-         responseFile(w,r,value)
-      case "official":
-         responseOfficial(w,r,value)
-         /*
-      case "bmclapi":
-         responseBMCLAPI(w,r,value)
-      case "default":
-         responseDefault(w,r,value)
-      case "gmclapi":
-         responseDefault(w,r,value)
-      */
-      default:
-         responseError(w,r,405,"Unspported choice for requests loading",10001)
-      }
-   }
-}
-func databaseService()  {
-   for {
-      defer recover()
-      file,err:=os.OpenFile(WorkPath+"/database.db",os.O_RDONLY,0600)
-      defer file.Close()
-      if err != nil {
-         panic(err)
-      }
-      bytes,err:=ioutil.ReadAll(file)
-      if err != nil {
-         panic(err)
-      }
-      err=json.Unmarshal(bytes,&Database)
-      if err != nil {
-         panic(err)
-      }
-      time.Sleep(time.Duration(10)*time.Second)
-   }
-}
-func nodeService()  {
-
-}
-func determineNode(hash string,fileName string) string {
-   return "http://127.0.0.1:8000/file/"+fileName+"/"+hash
-}
-func syncService()  {
-   for {
-   }
-}
-func hashFile(path string) string {
-   file,err:=os.Open(path)
-	defer file.Close()
-	if err != nil {
-		logIn("[Hash Thread]","Failed to calc hash of the file ",path,"! Error info: ",err.Error())
-	}
-	hash:=sha256.New()
-	if _,err := io.Copy(hash,file); err != nil {
-		logIn("[Hash Thread]","Failed to calc hash of the file ",path,"! Error info: ",err.Error())
-	}
-	sum:=hash.Sum(nil)
-   return hex.EncodeToString(sum)
-}
-func logIn(text ...string)  {
-   result:=""
-   for _,arg := range text {
-        result+=arg
-   }
-   log.Print(result)
-   Loger.Print(result)
-}
-func init()  {
-   WorkPath="gmclapi/"
-   DataPath=WorkPath+"data/"
-   err:=os.MkdirAll(WorkPath, os.ModePerm)
-   if err != nil {
-      panic(err)
-   }
-   err=os.MkdirAll(DataPath, os.ModePerm)
-   if err != nil {
-      panic(err)
-   }
-   if _,err:=os.Stat(WorkPath+"database.db"); os.IsNotExist(err){
-      database,err:=os.OpenFile(WorkPath+"database.db", os.O_WRONLY|os.O_CREATE, 0666)
-      database.Write([]byte("[]"))
-      if err != nil {
-         panic(err)
-      }
-      defer database.Close()
-   }
-   file,err:=os.OpenFile(WorkPath+"/database.db", os.O_RDONLY,0600)
-   defer file.Close()
-   if err != nil {
-      panic(err)
-   }
-   bytes,err:=ioutil.ReadAll(file)
-   if err != nil {
-      panic(err)
-   }
-   err=json.Unmarshal(bytes,&Database)
-   if err != nil {
-      panic(err)
-   }
-}
-func main()  {
-   logFile,err:=os.OpenFile(WorkPath+"logs.log", os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666)
-   if err != nil {
-      panic(err)
-   }
-   defer logFile.Close()
-   Loger=log.New(logFile,"",log.LstdFlags|log.Lshortfile)
-   http.HandleFunc("/", pageService)
-   go http.ListenAndServe("127.0.0.1:8000", nil)
-   go syncService()
-   go databaseService()
-   /*
-   go nodeService()
-   go consoleService()
-   */
-   for {
-
-   }
-}
-/*
-10001 Unspported choice for requests loading
-10002 Directory listing is not allowed
-10003 Cannot find file
-10004 The file exists but cannot be read
-10005 Please offer more params
-*/

master/src/database/database.go

@@ -0,0 +1 @@
+package database

master/src/main/bugs.md

@@ -0,0 +1 @@
+Log不能被正常记录

master/src/main/main.go

@@ -0,0 +1,80 @@
+/*
+Copyright Ghink Network Studio
+Website: https://www.ghink.net
+*/
+
+package main
+ 
+import (
+   "net/http"
+   "time"
+   "encoding/json"
+   "io/ioutil"
+   "os"
+   "io"
+   "crypto/sha256"
+   "encoding/hex"
+   "vars"
+   "page/service"
+   "tools/loger"
+)
+
+func databaseService()  {
+   for {
+      defer recover()
+      file,err:=os.OpenFile(vars.WorkPath+"/database.db",os.O_RDONLY,0600)
+      defer file.Close()
+      if err != nil {
+         continue
+      }
+      bytes,err:=ioutil.ReadAll(file)
+      if err != nil {
+         continue
+      }
+      err=json.Unmarshal(bytes,&vars.Database)
+      if err != nil {
+         continue
+      }
+      time.Sleep(time.Duration(10)*time.Second)
+   }
+}
+func nodeService()  {
+
+}
+func syncService()  {
+   for {
+   }
+}
+func hashFile(path string) string {
+   file,err:=os.Open(path)
+	defer file.Close()
+	if err != nil {
+		loger.LogIn("[Hash Thread]","Failed to calc hash of the file ",path,"! Error info: ",err.Error())
+	}
+	hash:=sha256.New()
+	if _,err := io.Copy(hash,file); err != nil {
+		loger.LogIn("[Hash Thread]","Failed to calc hash of the file ",path,"! Error info: ",err.Error())
+	}
+	sum:=hash.Sum(nil)
+   return hex.EncodeToString(sum)
+}
+func main()  {
+   http.HandleFunc("/", service.PageService)
+   go http.ListenAndServe(":8000", nil)
+   go syncService()
+   go databaseService()
+   /*
+   go nodeService()
+   go consoleService()
+   */
+   for {
+
+   }
+}
+/*
+10001 Unspported choice for requests loading
+10002 Directory listing is not allowed
+10003 Cannot find file
+10004 The file exists but cannot be read
+10005 Please offer more params
+*/

master/src/main/sync.py

@@ -24,8 +24,11 @@ def log(info):
     with open(workPath+"logs.log","a+") as fb:
         fb.write(info+"\n")
 def database():
-    threadLock[1]+=1
     global db
+    while True:
+        if threadLock[1]<threadLock[0]:
+            break
+    threadLock[1]+=1
     with open(workPath+"database.db","r") as fb:
         db=json.loads(fb.read())
     while True:
@@ -66,9 +69,16 @@ def syncMain():
             except:
                 pass
 def syncJson():
+    while True:
+        if threadLock[1]<threadLock[0]:
+            break
+    threadLock[1]+=1
     synced=[]
     def syncThread(rec):
         global db
+        while True:
+            if threadLock[1]<threadLock[0]:
+                break
         threadLock[1]+=1
         with open(dataPath+rec["hash"],"r") as fb:
             fall=re.findall('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+',fb.read())
@@ -109,16 +119,23 @@ def syncJson():
                     threading.Thread(target=syncThread,args=(rec,)).start()
         if i==0:
             break
-    log("Synchronizing for json list finished")
+    threadLock[1]-=1
 def syncForge(entrance="https://files.minecraftforge.net/net/minecraftforge/forge/"):
     global db
+    while True:
+        if threadLock[1]<threadLock[0]:
+            break
+    threadLock[1]+=1
     crawed=[]
     def syncThread(entrance):
         global db
+        while True:
+            if threadLock[1]<threadLock[0]:
+                break
         threadLock[1]+=1
         log("Crawling the page {}".format(entrance))
         page=requests.get(entrance,headers=userAgent,proxies=proxies).text
-        soup=bs(page,features="html5lib")
+        soup=bs(page,features="html.parser")
         pageurls=soup.find_all("a",href=True)
         for obj in pageurls:
             link=obj.get("href")
@@ -173,18 +190,21 @@ def syncForge(entrance="https://files.minecraftforge.net/net/minecraftforge/forg
                             time.sleep(10)
         threadLock[1]-=1
     syncThread(entrance)
-def monitor():
+    threadLock[1]-=1
+def serviceManager():
     while True:
-        log(str(threadLock))
-        time.sleep(1)
-def main():
-    threading.Thread(target=database).start()
-    #threading.Thread(target=monitor).start()
-    time.sleep(1)
-    while True:
-        time.sleep(60*60*24)
         syncMain()
         threading.Thread(target=syncJson()).start()
         threading.Thread(target=syncForge()).start()
+        time.sleep(60*60*24)
+def main():
+    threadLock[1]+=1
+    threading.Thread(target=database).start()
+    time.sleep(1)
+    threading.Thread(target=serviceManager).start()
+    while True:
+        command=input()
+        if command=="thread":
+            log("There's "+str(threadLock[1])+" threads are running, totally limited set as "+str(threadLock[0]))
 if __name__=="__main__":
     main()

master/src/page/response/errors/errors.go

@@ -0,0 +1,19 @@
+package errors
+
+import (
+	"net/http"
+	"encoding/json"
+)
+
+func ResponseError(w http.ResponseWriter,r *http.Request,httpCode int,info string,errorCode int)  {
+	w.Header().Set("Content-Type","application/json")
+	w.WriteHeader(httpCode)
+	type returnError struct{
+	   HttpCode int `json:"httpCode"`
+	   Info string `json:"info"`
+	   ErrorCode int `json:"errorCode"`
+	}
+	returnInfo:=returnError{httpCode,info,errorCode}
+	b,_:=json.Marshal(returnInfo)
+	w.Write(b)
+}

master/src/page/response/file/file.go

@@ -0,0 +1,42 @@
+package file
+
+import (
+	"net/http"
+	"strings"
+	"io/ioutil"
+	"os"
+	"page/response/errors"
+	"tools/loger"
+	"vars"
+)
+
+func ResponseFile(w http.ResponseWriter,r *http.Request,value []string)  {
+	test:=""
+	for _,arg := range value{
+	   test=test+"/"+arg
+	}
+	if test==""{
+	   errors.ResponseError(w,r,403,"Directory listing is not allowed",10002)
+	}else{
+	   fileName:=value[0]
+	   path:=vars.DataPath
+	   for _,arg := range value[1:]{
+		  path=path+"/"+arg
+	   }
+	   b,err:=ioutil.ReadFile(path)
+	   if err != nil {
+		  if find := strings.Contains(err.Error(),"The system cannot find"); find{
+			errors.ResponseError(w,r,404,"Cannot find file",10003)
+		  }else if s,_:=os.Stat(path); s.IsDir(){
+			errors.ResponseError(w,r,403,"Directory listing is not allowed",10002)
+		  }else{
+			errors.ResponseError(w,r,500,"The file exists but cannot be read",10004)
+			loger.LogIn("[Page Thread]","Read the file ",path," failed in data dir! Error info: ",err.Error())
+		  }
+	   }else{
+		  w.Header().Set("Content-Disposition","attachment;filename="+fileName)
+		  w.Header().Set("Content-Type","application/octet-stream")
+		  w.Write(b)
+	   }
+	}
+ }

master/src/page/response/icon/icon.go

@@ -0,0 +1,10 @@
+package icon
+
+import (
+	"net/http"
+)
+
+func ResponseICON(w http.ResponseWriter,r *http.Request)  {
+	w.Header().Set("Location","https://resource.ghink.net/public/img/logo/ghink.png")
+	w.WriteHeader(301)
+ }

master/src/page/response/official/official.go

@@ -0,0 +1,32 @@
+package official
+
+import (
+	"net/http"
+	"page/response/errors"
+	"vars"
+	"path"
+)
+
+func ResponseOfficial(w http.ResponseWriter,r *http.Request,value []string)  {
+	paths:=""
+	for _,arg := range value{
+	   paths=paths+"/"+arg
+	}
+	if paths=="" || paths=="/"{
+	   errors.ResponseError(w,r,400,"Please offer more params",10005)
+	}else{
+	   switcher:=true
+	   for _,arg := range vars.Database{
+		  if arg["path"]==paths{
+			 address:=vars.DetermineNode(arg["hash"],path.Base(arg["path"]))
+			 w.Header().Set("Location",address)
+			 w.WriteHeader(302)
+			 switcher=false
+			 break
+		  }
+	   }
+	   if switcher==true{
+			errors.ResponseError(w,r,404,"Cannot find file",10003)
+	   }
+	}
+ }

master/src/page/response/wiki/wiki.go

@@ -0,0 +1,18 @@
+package wiki
+
+import (
+	"net/http"
+	"encoding/json"
+)
+
+func ResponseWiKi(w http.ResponseWriter,r *http.Request)  {
+	w.Header().Set("Content-Type","application/json")
+	w.WriteHeader(200)
+	type returnWiki struct{
+	   HttpCode int `json:"httpCode"`
+	   Wiki string `json:"wiki"`
+	}
+	returnInfo:=returnWiki{200,"https://gitee.com/ghink/gmclapi/wikis"}
+	b,_:=json.Marshal(returnInfo)
+	w.Write(b)
+}

master/src/page/service/service.go

@@ -0,0 +1,48 @@
+package service
+
+import (
+	"net/http"
+	"strings"
+	"time"
+	"fmt"
+	"tools/loger"
+	"page/response/icon"
+	"page/response/wiki"
+	"page/response/file"
+	"page/response/official"
+	"page/response/errors"
+)
+
+func PageService(w http.ResponseWriter,r *http.Request) {
+	ts := time.Now().Unix()
+	loger.LogIn("[Page Thread]","Got requests from ",r.RemoteAddr,", address is ",r.URL.RequestURI())
+	w.Header().Set("Server","GMCLAPI (Alpha 1.0.0)")
+	w.Header().Set("Engine","Panda (Alpha 1.0.0)")
+	w.Header().Set("Stamp",fmt.Sprint(ts))
+	addr:=r.URL.RequestURI()
+	value:=strings.Split(addr,"/")
+	pathType:=value[1]
+	value=value[2:]
+	if addr=="/"{
+	   wiki.ResponseWiKi(w,r)
+	}else{
+	   switch pathType {
+	   case "favicon.ico":
+		  icon.ResponseICON(w,r)
+	   case "file":
+		  file.ResponseFile(w,r,value)
+	   case "official":
+		official.ResponseOfficial(w,r,value)
+		  /*
+	   case "bmclapi":
+		  responseBMCLAPI(w,r,value)
+	   case "default":
+		  responseDefault(w,r,value)
+	   case "gmclapi":
+		  responseDefault(w,r,value)
+	   */
+	   default:
+		  errors.ResponseError(w,r,405,"Unspported choice for requests loading",10001)
+	   }
+	}
+ }

master/src/tools/loger/loger.go

@@ -0,0 +1,27 @@
+package loger
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"vars"
+)
+
+var Loger *log.Logger
+
+func init()  {
+	logFile,err:=os.OpenFile(vars.WorkPath+"logs.log", os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666)
+	if err != nil {
+	   panic(err)
+	}
+	defer logFile.Close()
+	Loger=log.New(logFile,"",log.LstdFlags|log.Lshortfile)
+}
+func LogIn(text ...interface{})  {
+	result:=""
+	for _,arg := range text {
+		 result+=fmt.Sprint(arg)
+	}
+	log.Print(result)
+	Loger.Print(result)
+ }

master/src/vars/vars.go

@@ -0,0 +1,50 @@
+package vars
+
+import(
+	"os"
+	"io/ioutil"
+	"encoding/json"
+)
+
+var WorkPath,DataPath string
+var NodeList,Database,Dataread []map[string]string
+
+func init()  {
+	WorkPath="gmclapi/"
+	DataPath=WorkPath+"data/"
+	err:=os.MkdirAll(WorkPath, os.ModePerm)
+	if err != nil {
+	   panic(err)
+	}
+	err=os.MkdirAll(DataPath, os.ModePerm)
+	if err != nil {
+	   panic(err)
+	}
+	if _,err:=os.Stat(WorkPath+"database.db"); os.IsNotExist(err){
+	   database,err:=os.OpenFile(WorkPath+"database.db", os.O_WRONLY|os.O_CREATE, 0666)
+	   database.Write([]byte("[]"))
+	   if err != nil {
+		  panic(err)
+	   }
+	   defer database.Close()
+	}
+	for {
+	   file,err:=os.OpenFile(WorkPath+"/database.db", os.O_RDONLY,0600)
+	   defer file.Close()
+	   if err != nil {
+		  continue
+	   }
+	   bytes,err:=ioutil.ReadAll(file)
+	   if err != nil {
+		  continue
+	   }
+	   err=json.Unmarshal(bytes,&Database)
+	   if err != nil {
+		  continue
+	   }
+	   break
+	}
+}
+func DetermineNode(hash string,fileName string) string {
+   return "http://1.cn.server.ghink.net:8000/file/"+fileName+"/"+hash
+}