Add Status Updates whilst Gitea migrations are occurring (#15076)

* Add migrating message

Signed-off-by: Andrew Thornton <art27@cantab.net>

* simplify messenger

Signed-off-by: Andrew Thornton <art27@cantab.net>

* make messenger an interface

Signed-off-by: Andrew Thornton <art27@cantab.net>

* rename

Signed-off-by: Andrew Thornton <art27@cantab.net>

* prepare for merge

Signed-off-by: Andrew Thornton <art27@cantab.net>

* as per tech

Signed-off-by: Andrew Thornton <art27@cantab.net>

Co-authored-by: 6543 <6543@obermui.de>

models/migrations/migrations.go

@@ -317,6 +317,8 @@ var migrations = []Migration{
 	NewMigration("Add issue resource index table", addIssueResourceIndexTable),
 	// v183 -> v184
 	NewMigration("Create PushMirror table", createPushMirrorTable),
+	// v184 -> v185
+	NewMigration("Rename Task errors to message", renameTaskErrorsToMessage),
 }
 
 // GetCurrentDBVersion returns the current db version

models/migrations/v184.go

@@ -0,0 +1,47 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+	"fmt"
+
+	"code.gitea.io/gitea/modules/setting"
+
+	"xorm.io/xorm"
+)
+
+func renameTaskErrorsToMessage(x *xorm.Engine) error {
+	type Task struct {
+		Errors string `xorm:"TEXT"` // if task failed, saved the error reason
+		Type   int
+		Status int `xorm:"index"`
+	}
+
+	sess := x.NewSession()
+	defer sess.Close()
+	if err := sess.Begin(); err != nil {
+		return err
+	}
+
+	if err := sess.Sync2(new(Task)); err != nil {
+		return fmt.Errorf("error on Sync2: %v", err)
+	}
+
+	switch {
+	case setting.Database.UseMySQL:
+		if _, err := sess.Exec("ALTER TABLE `task` CHANGE errors message text"); err != nil {
+			return err
+		}
+	case setting.Database.UseMSSQL:
+		if _, err := sess.Exec("sp_rename 'task.errors', 'message', 'COLUMN'"); err != nil {
+			return err
+		}
+	default:
+		if _, err := sess.Exec("ALTER TABLE `task` RENAME COLUMN errors TO message"); err != nil {
+			return err
+		}
+	}
+	return sess.Commit()
+}

models/task.go

@@ -32,10 +32,16 @@ type Task struct {
 	StartTime      timeutil.TimeStamp
 	EndTime        timeutil.TimeStamp
 	PayloadContent string             `xorm:"TEXT"`
-	Errors         string             `xorm:"TEXT"` // if task failed, saved the error reason
+	Message        string             `xorm:"TEXT"` // if task failed, saved the error reason
 	Created        timeutil.TimeStamp `xorm:"created"`
 }
 
+// TranslatableMessage represents JSON struct that can be translated with a Locale
+type TranslatableMessage struct {
+	Format string
+	Args   []interface{} `json:"omitempty"`
+}
+
 // LoadRepo loads repository of the task
 func (task *Task) LoadRepo() error {
 	return task.loadRepo(x)

modules/migrations/base/messenger.go

@@ -0,0 +1,11 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package base
+
+// Messenger is a formatting function similar to i18n.Tr
+type Messenger func(key string, args ...interface{})
+
+// NilMessenger represents an empty formatting function
+func NilMessenger(string, ...interface{}) {}

modules/migrations/dump.go

@@ -555,7 +555,7 @@ func DumpRepository(ctx context.Context, baseDir, ownerName string, opts base.Mi
 		return err
 	}
 
-	if err := migrateRepository(downloader, uploader, opts); err != nil {
+	if err := migrateRepository(downloader, uploader, opts, nil); err != nil {
 		if err1 := uploader.Rollback(); err1 != nil {
 			log.Error("rollback failed: %v", err1)
 		}
@@ -620,7 +620,7 @@ func RestoreRepository(ctx context.Context, baseDir string, ownerName, repoName
 	}
 	updateOptionsUnits(&migrateOpts, units)
 
-	if err = migrateRepository(downloader, uploader, migrateOpts); err != nil {
+	if err = migrateRepository(downloader, uploader, migrateOpts, nil); err != nil {
 		if err1 := uploader.Rollback(); err1 != nil {
 			log.Error("rollback failed: %v", err1)
 		}

modules/migrations/gitea_uploader_test.go

@@ -47,7 +47,7 @@ func TestGiteaUploadRepo(t *testing.T) {
 		PullRequests: true,
 		Private:      true,
 		Mirror:       false,
-	})
+	}, nil)
 	assert.NoError(t, err)
 
 	repo := models.AssertExistsAndLoadBean(t, &models.Repository{OwnerID: user.ID, Name: repoName}).(*models.Repository)

modules/migrations/migrate.go

@@ -99,7 +99,7 @@ func IsMigrateURLAllowed(remoteURL string, doer *models.User) error {
 }
 
 // MigrateRepository migrate repository according MigrateOptions
-func MigrateRepository(ctx context.Context, doer *models.User, ownerName string, opts base.MigrateOptions) (*models.Repository, error) {
+func MigrateRepository(ctx context.Context, doer *models.User, ownerName string, opts base.MigrateOptions, messenger base.Messenger) (*models.Repository, error) {
 	err := IsMigrateURLAllowed(opts.CloneAddr, doer)
 	if err != nil {
 		return nil, err
@@ -118,7 +118,7 @@ func MigrateRepository(ctx context.Context, doer *models.User, ownerName string,
 	var uploader = NewGiteaLocalUploader(ctx, doer, ownerName, opts.RepoName)
 	uploader.gitServiceType = opts.GitServiceType
 
-	if err := migrateRepository(downloader, uploader, opts); err != nil {
+	if err := migrateRepository(downloader, uploader, opts, messenger); err != nil {
 		if err1 := uploader.Rollback(); err1 != nil {
 			log.Error("rollback failed: %v", err1)
 		}
@@ -167,7 +167,11 @@ func newDownloader(ctx context.Context, ownerName string, opts base.MigrateOptio
 // migrateRepository will download information and then upload it to Uploader, this is a simple
 // process for small repository. For a big repository, save all the data to disk
 // before upload is better
-func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts base.MigrateOptions) error {
+func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts base.MigrateOptions, messenger base.Messenger) error {
+	if messenger == nil {
+		messenger = base.NilMessenger
+	}
+
 	repo, err := downloader.GetRepoInfo()
 	if err != nil {
 		if !base.IsErrNotSupported(err) {
@@ -185,12 +189,14 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
 	}
 
 	log.Trace("migrating git data from %s", repo.CloneURL)
+	messenger("repo.migrate.migrating_git")
 	if err = uploader.CreateRepo(repo, opts); err != nil {
 		return err
 	}
 	defer uploader.Close()
 
 	log.Trace("migrating topics")
+	messenger("repo.migrate.migrating_topics")
 	topics, err := downloader.GetTopics()
 	if err != nil {
 		if !base.IsErrNotSupported(err) {
@@ -206,6 +212,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
 
 	if opts.Milestones {
 		log.Trace("migrating milestones")
+		messenger("repo.migrate.migrating_milestones")
 		milestones, err := downloader.GetMilestones()
 		if err != nil {
 			if !base.IsErrNotSupported(err) {
@@ -229,6 +236,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
 
 	if opts.Labels {
 		log.Trace("migrating labels")
+		messenger("repo.migrate.migrating_labels")
 		labels, err := downloader.GetLabels()
 		if err != nil {
 			if !base.IsErrNotSupported(err) {
@@ -252,6 +260,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
 
 	if opts.Releases {
 		log.Trace("migrating releases")
+		messenger("repo.migrate.migrating_releases")
 		releases, err := downloader.GetReleases()
 		if err != nil {
 			if !base.IsErrNotSupported(err) {
@@ -285,6 +294,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
 
 	if opts.Issues {
 		log.Trace("migrating issues and comments")
+		messenger("repo.migrate.migrating_issues")
 		var issueBatchSize = uploader.MaxBatchInsertSize("issue")
 
 		for i := 1; ; i++ {
@@ -339,6 +349,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
 
 	if opts.PullRequests {
 		log.Trace("migrating pull requests and comments")
+		messenger("repo.migrate.migrating_pulls")
 		var prBatchSize = uploader.MaxBatchInsertSize("pullrequest")
 		for i := 1; ; i++ {
 			prs, isEnd, err := downloader.GetPullRequests(i, prBatchSize)

modules/task/migrate.go

@@ -20,6 +20,7 @@ import (
 	"code.gitea.io/gitea/modules/structs"
 	"code.gitea.io/gitea/modules/timeutil"
 	"code.gitea.io/gitea/modules/util"
+	jsoniter "github.com/json-iterator/go"
 )
 
 func handleCreateError(owner *models.User, err error) error {
@@ -56,7 +57,7 @@ func runMigrateTask(t *models.Task) (err error) {
 
 		t.EndTime = timeutil.TimeStampNow()
 		t.Status = structs.TaskStatusFailed
-		t.Errors = err.Error()
+		t.Message = err.Error()
 		t.RepoID = 0
 		if err := t.UpdateCols("status", "errors", "repo_id", "end_time"); err != nil {
 			log.Error("Task UpdateCols failed: %v", err)
@@ -106,7 +107,16 @@ func runMigrateTask(t *models.Task) (err error) {
 		return
 	}
 
-	repo, err = migrations.MigrateRepository(ctx, t.Doer, t.Owner.Name, *opts)
+	repo, err = migrations.MigrateRepository(ctx, t.Doer, t.Owner.Name, *opts, func(format string, args ...interface{}) {
+		message := models.TranslatableMessage{
+			Format: format,
+			Args:   args,
+		}
+		json := jsoniter.ConfigCompatibleWithStandardLibrary
+		bs, _ := json.Marshal(message)
+		t.Message = string(bs)
+		_ = t.UpdateCols("message")
+	})
 	if err == nil {
 		log.Trace("Repository migrated [%d]: %s/%s", repo.ID, t.Owner.Name, repo.Name)
 		return

options/locale/locale_en-US.ini

@@ -824,11 +824,19 @@ migrated_from_fake = Migrated From %[1]s
 migrate.migrate = Migrate From %s
 migrate.migrating = Migrating from <b>%s</b> ...
 migrate.migrating_failed = Migrating from <b>%s</b> failed.
+migrate.migrating_failed.error = Error: %s
 migrate.github.description = Migrating data from Github.com or Github Enterprise.
 migrate.git.description = Migrating or Mirroring git data from Git services
 migrate.gitlab.description = Migrating data from GitLab.com or Self-Hosted gitlab server.
 migrate.gitea.description = Migrating data from Gitea.com or Self-Hosted Gitea server.
 migrate.gogs.description = Migrating data from notabug.org or other Self-Hosted Gogs server.
+migrate.migrating_git = Migrating Git Data
+migrate.migrating_topics = Migrating Topics
+migrate.migrating_milestones = Migrating Milestones
+migrate.migrating_labels = Migrating Labels
+migrate.migrating_releases = Migrating Releases
+migrate.migrating_issues = Migrating Issues
+migrate.migrating_pulls = Migrating Pull Requests
 
 mirror_from = mirror of
 forked_from = forked from

routers/api/v1/repo/migrate.go

@@ -199,7 +199,7 @@ func Migrate(ctx *context.APIContext) {
 		}
 	}()
 
-	if _, err = migrations.MigrateRepository(graceful.GetManager().HammerContext(), ctx.User, repoOwner.Name, opts); err != nil {
+	if _, err = migrations.MigrateRepository(graceful.GetManager().HammerContext(), ctx.User, repoOwner.Name, opts, nil); err != nil {
 		handleMigrateError(ctx, repoOwner, remoteAddr, err)
 		return
 	}

routers/web/user/task.go

@@ -9,6 +9,7 @@ import (
 
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/context"
+	jsoniter "github.com/json-iterator/go"
 )
 
 // TaskStatus returns task's status
@@ -21,9 +22,24 @@ func TaskStatus(ctx *context.Context) {
 		return
 	}
 
+	message := task.Message
+
+	if task.Message != "" && task.Message[0] == '{' {
+		// assume message is actually a translatable string
+		json := jsoniter.ConfigCompatibleWithStandardLibrary
+		var translatableMessage models.TranslatableMessage
+		if err := json.Unmarshal([]byte(message), &translatableMessage); err != nil {
+			translatableMessage = models.TranslatableMessage{
+				Format: "migrate.migrating_failed.error",
+				Args:   []interface{}{task.Message},
+			}
+		}
+		message = ctx.Tr(translatableMessage.Format, translatableMessage.Args...)
+	}
+
 	ctx.JSON(http.StatusOK, map[string]interface{}{
 		"status":    task.Status,
-		"err":       task.Errors,
+		"message":   message,
 		"repo-id":   task.RepoID,
 		"repo-name": opts.RepoName,
 		"start":     task.StartTime,

templates/repo/migrate/migrating.tmpl

@@ -22,6 +22,7 @@
 						<div class="sixteen wide center aligned centered column">
 							<div id="repo_migrating_progress">
 								<p>{{.i18n.Tr "repo.migrate.migrating" .CloneAddr | Safe}}</p>
+								<p id="repo_migrating_progress_message"></p>
 							</div>
 							<div id="repo_migrating_failed" hidden>
 								<p>{{.i18n.Tr "repo.migrate.migrating_failed" .CloneAddr | Safe}}</p>

web_src/js/index.js

@@ -202,6 +202,7 @@ function initRepoStatusChecker() {
   const migrating = $('#repo_migrating');
   $('#repo_migrating_failed').hide();
   $('#repo_migrating_failed_image').hide();
+  $('#repo_migrating_progress_message').hide();
   if (migrating) {
     const task = migrating.attr('task');
     if (typeof task === 'undefined') {
@@ -223,9 +224,13 @@ function initRepoStatusChecker() {
             $('#repo_migrating').hide();
             $('#repo_migrating_failed').show();
             $('#repo_migrating_failed_image').show();
-            $('#repo_migrating_failed_error').text(xhr.responseJSON.err);
+            $('#repo_migrating_failed_error').text(xhr.responseJSON.message);
             return;
           }
+          if (xhr.responseJSON.message) {
+            $('#repo_migrating_progress_message').show();
+            $('#repo_migrating_progress_message').text(xhr.responseJSON.message);
+          }
           setTimeout(() => {
             initRepoStatusChecker();
           }, 2000);