first version: v0.0.1 Alpha

.gitignore

@@ -46,7 +46,8 @@ $RECYCLE.BIN/
 .LSOverride
 
 # Icon must end with two \r
-Icon
+Icon
+
 
 # Thumbnails
 ._*
@@ -485,31 +486,34 @@ FodyWeavers.xsd
 # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider
 # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
 
+# Whole .idea
+.idea
+
 # User-specific stuff
-.idea/**/workspace.xml
-.idea/**/tasks.xml
-.idea/**/usage.statistics.xml
-.idea/**/dictionaries
-.idea/**/shelf
+#.idea/**/workspace.xml
+#.idea/**/tasks.xml
+#.idea/**/usage.statistics.xml
+#.idea/**/dictionaries
+#.idea/**/shelf
 
 # AWS User-specific
-.idea/**/aws.xml
+#.idea/**/aws.xml
 
 # Generated files
-.idea/**/contentModel.xml
+#.idea/**/contentModel.xml
 
 # Sensitive or high-churn files
-.idea/**/dataSources/
-.idea/**/dataSources.ids
-.idea/**/dataSources.local.xml
-.idea/**/sqlDataSources.xml
-.idea/**/dynamic.xml
-.idea/**/uiDesigner.xml
-.idea/**/dbnavigator.xml
+#.idea/**/dataSources/
+#.idea/**/dataSources.ids
+#.idea/**/dataSources.local.xml
+#.idea/**/sqlDataSources.xml
+#.idea/**/dynamic.xml
+#.idea/**/uiDesigner.xml
+#.idea/**/dbnavigator.xml
 
 # Gradle
-.idea/**/gradle.xml
-.idea/**/libraries
+#.idea/**/gradle.xml
+#.idea/**/libraries
 
 # Gradle and Maven with auto-import
 # When using Gradle or Maven with auto-import, you should exclude module files,
@@ -528,7 +532,7 @@ FodyWeavers.xsd
 cmake-build-*/
 
 # Mongo Explorer plugin
-.idea/**/mongoSettings.xml
+#.idea/**/mongoSettings.xml
 
 # File-based project format
 *.iws
@@ -537,16 +541,16 @@ cmake-build-*/
 out/
 
 # mpeltonen/sbt-idea plugin
-.idea_modules/
+#.idea_modules/
 
 # JIRA plugin
 atlassian-ide-plugin.xml
 
 # Cursive Clojure plugin
-.idea/replstate.xml
+#.idea/replstate.xml
 
 # SonarLint plugin
-.idea/sonarlint/
+#.idea/sonarlint/
 
 # Crashlytics plugin (for Android Studio and IntelliJ)
 com_crashlytics_export_strings.xml
@@ -555,8 +559,8 @@ crashlytics-build.properties
 fabric.properties
 
 # Editor-based Rest Client
-.idea/httpRequests
+#.idea/httpRequests
 
 # Android studio 3.1+ serialized cache file
-.idea/caches/build_file_checksums.ser
+#.idea/caches/build_file_checksums.ser
 

controller.go

@@ -0,0 +1,73 @@
+package richkago
+
+import "sync"
+
+type Controller struct {
+	paused           bool
+	excepted         bool
+	totalSize        int64
+	downloadedSize   int64
+	downloadedSlices map[string]int64
+	mu               sync.Mutex
+}
+
+// NewController build a new controller
+func NewController() *Controller {
+	return &Controller{
+		downloadedSlices: make(map[string]int64),
+	}
+}
+
+// UpdateProgress update progress into controller
+func (c *Controller) UpdateProgress(size int64, chunkID string) {
+	// Get lock
+	c.mu.Lock()
+	defer c.mu.Unlock()
+
+	if chunkID == "" && len(c.downloadedSlices) == 0 {
+		// Init variable
+		c.downloadedSize = size
+	} else {
+		// Update progress
+		c.downloadedSlices[chunkID] = size
+		c.downloadedSize = 0
+		// Sum up
+		for _, v := range c.downloadedSlices {
+			c.downloadedSize += v
+		}
+	}
+}
+
+// Pause pause a progress
+func (c *Controller) Pause() {
+	c.paused = true
+}
+
+// Unpause unpause a progress
+func (c *Controller) Unpause() {
+	c.paused = false
+}
+
+// Status gets a status of a controller
+func (c *Controller) Status() int {
+	if c.downloadedSize == 0 && !c.excepted {
+		return -1 // Not started
+	} else if c.paused {
+		return -2 // Paused
+	} else if c.excepted {
+		return -3 // Excepted
+	} else if c.downloadedSize == c.totalSize {
+		return 0 // Done
+	} else {
+		return 1 // Downloading
+	}
+}
+
+// Progress gets progress of a controller
+func (c *Controller) Progress() float64 {
+	if c.totalSize == 0 {
+		return -1
+	}
+
+	return float64(c.downloadedSize) / float64(c.totalSize) * 100
+}

controller_test.go

@@ -0,0 +1,71 @@
+package richkago
+
+import "testing"
+
+// TestNewController test example that builds a new controller
+func TestNewController(t *testing.T) {
+	controller := NewController()
+	if controller == nil {
+		t.Error("controller is nil")
+		return
+	}
+}
+
+// TestController_UpdateProgress test example that updates progress
+func TestController_UpdateProgress(t *testing.T) {
+	controller := NewController()
+	controller.totalSize = 1000
+
+	controller.UpdateProgress(100, "1")
+	if controller.Progress() != 10 {
+		t.Error("progress is wrong", controller.Progress())
+		return
+	}
+
+	if controller.Status() != 1 {
+		t.Error("status is wrong", controller.Status())
+		return
+	}
+}
+
+// TestController_Pause test example that pause a progress
+func TestController_Pause(t *testing.T) {
+	controller := NewController()
+	controller.totalSize = 1000
+
+	controller.UpdateProgress(100, "1")
+	if controller.Progress() != 10 {
+		t.Error("progress is wrong", controller.Progress())
+		return
+	}
+
+	controller.Pause()
+	if controller.Status() != -2 {
+		t.Error("status is wrong", controller.Status())
+		return
+	}
+}
+
+// TestController_Unpause test example that unpause a progress
+func TestController_Unpause(t *testing.T) {
+	controller := NewController()
+	controller.totalSize = 1000
+
+	controller.UpdateProgress(100, "1")
+	if controller.Progress() != 10 {
+		t.Error("progress is wrong", controller.Progress())
+		return
+	}
+
+	controller.Pause()
+	if controller.Status() != -2 {
+		t.Error("status is wrong", controller.Status())
+		return
+	}
+
+	controller.Unpause()
+	if controller.Status() != 1 {
+		t.Error("status is wrong", controller.Status())
+		return
+	}
+}

go.mod

@@ -0,0 +1,5 @@
+module github.com/ghinknet/richkago
+
+go 1.23.2
+
+require golang.org/x/sync v0.9.0

go.sum

@@ -0,0 +1,2 @@
+golang.org/x/sync v0.9.0 h1:fEo0HyrW1GIgZdpbhCRO0PkJajUS5H9IFUztCgEo2jQ=
+golang.org/x/sync v0.9.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=

richkago.go

@@ -0,0 +1,316 @@
+package richkago
+
+import (
+	"errors"
+	"fmt"
+	"golang.org/x/sync/errgroup"
+	"io"
+	"net/http"
+	"os"
+	"strconv"
+	"time"
+)
+
+const (
+	version        = "Alpha/0.0.1"
+	userAgent      = "Richkago" + version
+	coroutineLimit = 10
+	sliceThreshold = 10 * 1024 * 1024 // 10 MiB
+	timeout        = 3 * time.Second
+	retryTimes     = 5
+	chunkSize      = 102400
+)
+
+// readWithTimeout read data from source with timeout limit
+func readWithTimeout(reader io.Reader, buffer []byte, controller *Controller, file *os.File, downloaded *int64, chunkID string) error {
+	// Make a error chan
+	ch := make(chan error, 1)
+
+	go func() {
+		// Read data
+		n, err := reader.Read(buffer)
+		if n > 0 {
+			// Write to file
+			_, err = file.Write(buffer[:n])
+			if err != nil {
+				ch <- err
+				return
+			}
+			// Calc amount of downloaded data
+			*downloaded += int64(n)
+		}
+		ch <- err
+	}()
+
+	// Error and timeout handler
+	select {
+	case err := <-ch:
+		// Update amount of downloaded data
+		if controller != nil {
+			controller.UpdateProgress(*downloaded, chunkID)
+		}
+
+		return err
+	case <-time.After(timeout):
+		return errors.New("timeout while reading data")
+	}
+}
+
+// downloadRange download a file with many slices
+func downloadRange(client *http.Client, url string, start, end int64, destination string, controller *Controller) error {
+	// Build request header
+	headers := map[string]string{
+		"User-Agent": userAgent,
+		"Range":      fmt.Sprintf("bytes=%d-%d", start, end),
+	}
+
+	retries := retryTimes
+	var err error
+
+	for retries > 0 {
+		// Get header info
+		var req *http.Request
+		req, err = http.NewRequest("GET", url, nil)
+		if err != nil {
+			retries--
+			time.Sleep(1 * time.Second)
+			continue
+		}
+		for k, v := range headers {
+			req.Header.Add(k, v)
+		}
+
+		// Read http header
+		var resp *http.Response
+		resp, err = client.Do(req)
+		if err != nil || resp.StatusCode != http.StatusPartialContent {
+			retries--
+			time.Sleep(1 * time.Second)
+			continue
+		}
+
+		defer func(Body io.ReadCloser) {
+			err = Body.Close()
+			if err != nil {
+				panic(err)
+			}
+		}(resp.Body)
+
+		// Pre-create file
+		file, _ := os.OpenFile(destination, os.O_WRONLY, 0644)
+		defer func(file *os.File) {
+			err = file.Close()
+			if err != nil {
+				panic(err)
+			}
+		}(file)
+		_, err = file.Seek(start, io.SeekStart)
+		if err != nil {
+			retries--
+			time.Sleep(1 * time.Second)
+			continue
+		}
+
+		// Create goroutines to download
+		buffer := make([]byte, chunkSize)
+		var downloaded int64
+		for {
+			// Controller pin
+			if controller.paused {
+				time.Sleep(1 * time.Second)
+				continue
+			}
+
+			// Start read stream
+			err = readWithTimeout(resp.Body, buffer, controller, file, &downloaded, fmt.Sprintf("%d-%d", start, end))
+			if err == io.EOF {
+				break
+			} else if err != nil {
+				break
+			}
+		}
+
+		// Error handler
+		if err != nil && err != io.EOF {
+			retries--
+			time.Sleep(1 * time.Second)
+			continue
+		}
+
+		return nil
+	}
+
+	return err
+}
+
+// downloadSingle download a file directly
+func downloadSingle(client *http.Client, url, destination string, controller *Controller) error {
+	// Build request header
+	headers := map[string]string{
+		"User-Agent": userAgent,
+	}
+	retries := retryTimes
+
+	for retries > 0 {
+		// Get header info
+		req, _ := http.NewRequest("GET", url, nil)
+		for k, v := range headers {
+			req.Header.Add(k, v)
+		}
+
+		// Read http header
+		resp, err := client.Do(req)
+		if err != nil || resp.StatusCode != http.StatusOK {
+			retries--
+			time.Sleep(1 * time.Second)
+			continue
+		}
+
+		defer func(Body io.ReadCloser) {
+			err = Body.Close()
+			if err != nil {
+				panic(err)
+			}
+		}(resp.Body)
+
+		// Pre-create file
+		file, _ := os.OpenFile(destination, os.O_WRONLY, 0644)
+		defer func(file *os.File) {
+			err = file.Close()
+			if err != nil {
+				panic(err)
+			}
+		}(file)
+
+		// Start downloading
+		buffer := make([]byte, chunkSize)
+		var downloaded int64
+		for {
+			// Controller pin
+			if controller.paused {
+				time.Sleep(1 * time.Second)
+				continue
+			}
+
+			// Start read stream
+			var n int
+			n, err = resp.Body.Read(buffer)
+			if err != nil {
+				break
+			}
+			if n > 0 {
+				_, err = file.Write(buffer[:n])
+				if err != nil {
+					return err
+				}
+				downloaded += int64(n)
+				controller.UpdateProgress(downloaded, "")
+			}
+		}
+
+		// Error handler
+		if err != nil {
+			retries--
+			time.Sleep(1 * time.Second)
+			continue
+		} else {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// Download main download task creator
+func Download(url, destination string, controller *Controller) (float64, int64, error) {
+	// Create http client
+	client := &http.Client{}
+
+	// Get header info
+	req, err := http.NewRequest("HEAD", url, nil)
+	if err != nil {
+		controller.excepted = true
+		return 0, 0, err
+	}
+	req.Header.Add("User-Agent", userAgent)
+
+	// Read http header
+	resp, err := client.Do(req)
+	if err != nil {
+		controller.excepted = true
+		return 0, 0, err
+	}
+	defer func(Body io.ReadCloser) {
+		err = Body.Close()
+		if err != nil {
+			panic(err)
+		}
+	}(resp.Body)
+
+	sizeStr := resp.Header.Get("Content-Length")
+	fileSize, _ := strconv.ParseInt(sizeStr, 10, 64)
+	controller.totalSize = fileSize
+
+	// Calc slices size
+	if fileSize <= sliceThreshold {
+		file, _ := os.Create(destination)
+		err = file.Truncate(fileSize)
+		if err != nil {
+			controller.excepted = true
+			return 0, 0, err
+		}
+		err = file.Close()
+		if err != nil {
+			controller.excepted = true
+			return 0, 0, err
+		}
+
+		// Too small
+		startTime := time.Now()
+		err = downloadSingle(client, url, destination, controller)
+		if err != nil {
+			controller.excepted = true
+			return 0, 0, err
+		}
+		endTime := time.Now()
+		return endTime.Sub(startTime).Seconds(), fileSize, nil
+	}
+
+	// Pre-create file
+	partSize := fileSize / coroutineLimit
+	file, _ := os.Create(destination)
+	err = file.Truncate(fileSize)
+	if err != nil {
+		controller.excepted = true
+		return 0, 0, err
+	}
+	err = file.Close()
+	if err != nil {
+		controller.excepted = true
+		return 0, 0, err
+	}
+
+	// Start download goroutines
+	group := new(errgroup.Group)
+	for i := 0; i < coroutineLimit; i++ {
+		start := int64(i) * partSize
+		end := start + partSize - 1
+		if i == coroutineLimit-1 {
+			end = fileSize - 1
+		}
+		group.Go(func() error {
+			err = downloadRange(client, url, start, end, destination, controller)
+			return err
+		})
+	}
+
+	// Start all tasks
+	startTime := time.Now()
+	if err = group.Wait(); err != nil {
+		controller.excepted = true
+		return 0, 0, err
+	}
+	endTime := time.Now()
+
+	return endTime.Sub(startTime).Seconds(), fileSize, nil
+}

richkago_test.go

@@ -0,0 +1,16 @@
+package richkago
+
+import (
+	"testing"
+)
+
+// TestDownload test main download
+func TestDownload(t *testing.T) {
+	controller := NewController()
+
+	_, _, err := Download("https://mirrors.tuna.tsinghua.edu.cn/github-release/git-for-windows/git/LatestRelease/Git-2.47.0.2-64-bit.exe", "Git-2.47.0.2-64-bit.exe", controller)
+	if err != nil {
+		t.Error("failed to download", err)
+		return
+	}
+}