repobase/repobase
1
0
Fork 0
Code Issues 8 Pull requests Projects 1 Releases Packages Wiki Activity Actions 4

Move migrating repository from frontend to backend (#6200)

Browse source 
* move migrating to backend

* add loading image when migrating and fix tests

* fix format

* fix lint

* add redis task queue support and improve docs

* add redis vendor

* fix vet

* add database migrations and fix app.ini sample

* add comments for task section on app.ini.sample

* Update models/migrations/v84.go

Co-Authored-By: lunny <xiaolunwen@gmail.com>

* Update models/repo.go

Co-Authored-By: lunny <xiaolunwen@gmail.com>

* move migrating to backend

* add loading image when migrating and fix tests

* fix fmt

* add redis task queue support and improve docs

* fix fixtures

* fix fixtures

* fix duplicate function on index.js

* fix tests

* rename repository statuses

* check if repository is being create when SSH request

* fix lint

* fix template

* some improvements

* fix template

* unified migrate options

* fix lint

* fix loading page

* refactor

* When gitea restart, don't restart the running tasks because we may have servel gitea instances, that may break the migration

* fix js

* Update models/repo.go

Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com>

* Update docs/content/doc/advanced/config-cheat-sheet.en-us.md

Co-Authored-By: guillep2k <18600385+guillep2k@users.noreply.github.com>

* fix tests

* rename ErrTaskIsNotExist to ErrTaskDoesNotExist

* delete release after add one on tests to make it run happy

* fix tests

* fix tests

* improve codes

* fix lint

* fix lint

* fix migrations
This commit is contained in:
Lunny Xiao 2019-10-13 21:23:14 +08:00 • committed by GitHub
parent 0a96e59884
commit f2a3abc683
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
37 changed files with 1192 additions and 222 deletions
Download patch file Download diff file Expand all files Collapse all files

4
routers/api/v1/repo/repo.go
View file

@ -398,8 +398,8 @@ func Migrate(ctx *context.APIContext, form auth.MigrateRepoForm) {
}
var opts = migrations.MigrateOptions{
RemoteURL: remoteAddr,
Name: form.RepoName,
CloneAddr: remoteAddr,
RepoName: form.RepoName,
Description: form.Description,
Private: form.Private || setting.Repository.ForcePrivate,
Mirror: form.Mirror,

4
routers/init.go
View file

@ -20,6 +20,7 @@ import (
"code.gitea.io/gitea/modules/markup/external"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/ssh"
"code.gitea.io/gitea/modules/task"
"code.gitea.io/gitea/services/mailer"
mirror_service "code.gitea.io/gitea/services/mirror"
@ -102,6 +103,9 @@ func GlobalInit() {
mirror_service.InitSyncMirrors()
models.InitDeliverHooks()
models.InitTestPullRequests()
if err := task.Init(); err != nil {
log.Fatal("Failed to initialize task scheduler: %v", err)
}
}
if setting.EnableSQLite3 {
log.Info("SQLite3 Supported")

9
routers/private/serv.go
View file

@ -119,6 +119,15 @@ func ServCommand(ctx *macaron.Context) {
repo.OwnerName = ownerName
results.RepoID = repo.ID
if repo.IsBeingCreated() {
ctx.JSON(http.StatusInternalServerError, map[string]interface{}{
"results": results,
"type": "InternalServerError",
"err": "Repository is being created, you could retry after it finished",
})
return
}
// We can shortcut at this point if the repo is a mirror
if mode > models.AccessModeRead && repo.IsMirror {
ctx.JSON(http.StatusUnauthorized, map[string]interface{}{

103
routers/repo/repo.go
View file

@ -19,6 +19,7 @@ import (
"code.gitea.io/gitea/modules/migrations"
"code.gitea.io/gitea/modules/notification"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/task"
"code.gitea.io/gitea/modules/util"
"github.com/unknwon/com"
@ -133,8 +134,6 @@ func Create(ctx *context.Context) {
func handleCreateError(ctx *context.Context, owner *models.User, err error, name string, tpl base.TplName, form interface{}) {
switch {
case migrations.IsRateLimitError(err):
ctx.RenderWithErr(ctx.Tr("form.visit_rate_limit"), tpl, form)
case models.IsErrReachLimitOfRepo(err):
ctx.RenderWithErr(ctx.Tr("repo.form.reach_limit_of_creation", owner.MaxCreationLimit()), tpl, form)
case models.IsErrRepoAlreadyExist(err):
@ -221,6 +220,40 @@ func Migrate(ctx *context.Context) {
ctx.HTML(200, tplMigrate)
}
func handleMigrateError(ctx *context.Context, owner *models.User, err error, name string, tpl base.TplName, form *auth.MigrateRepoForm) {
switch {
case migrations.IsRateLimitError(err):
ctx.RenderWithErr(ctx.Tr("form.visit_rate_limit"), tpl, form)
case migrations.IsTwoFactorAuthError(err):
ctx.RenderWithErr(ctx.Tr("form.2fa_auth_required"), tpl, form)
case models.IsErrReachLimitOfRepo(err):
ctx.RenderWithErr(ctx.Tr("repo.form.reach_limit_of_creation", owner.MaxCreationLimit()), tpl, form)
case models.IsErrRepoAlreadyExist(err):
ctx.Data["Err_RepoName"] = true
ctx.RenderWithErr(ctx.Tr("form.repo_name_been_taken"), tpl, form)
case models.IsErrNameReserved(err):
ctx.Data["Err_RepoName"] = true
ctx.RenderWithErr(ctx.Tr("repo.form.name_reserved", err.(models.ErrNameReserved).Name), tpl, form)
case models.IsErrNamePatternNotAllowed(err):
ctx.Data["Err_RepoName"] = true
ctx.RenderWithErr(ctx.Tr("repo.form.name_pattern_not_allowed", err.(models.ErrNamePatternNotAllowed).Pattern), tpl, form)
default:
remoteAddr, _ := form.ParseRemoteAddr(owner)
err = util.URLSanitizedError(err, remoteAddr)
if strings.Contains(err.Error(), "Authentication failed") ||
strings.Contains(err.Error(), "Bad credentials") ||
strings.Contains(err.Error(), "could not read Username") {
ctx.Data["Err_Auth"] = true
ctx.RenderWithErr(ctx.Tr("form.auth_failed", err.Error()), tpl, form)
} else if strings.Contains(err.Error(), "fatal:") {
ctx.Data["Err_CloneAddr"] = true
ctx.RenderWithErr(ctx.Tr("repo.migrate.failed", err.Error()), tpl, form)
} else {
ctx.ServerError(name, err)
}
}
}
// MigratePost response for migrating from external git repository
func MigratePost(ctx *context.Context, form auth.MigrateRepoForm) {
ctx.Data["Title"] = ctx.Tr("new_migrate")
@ -258,8 +291,8 @@ func MigratePost(ctx *context.Context, form auth.MigrateRepoForm) {
}
var opts = migrations.MigrateOptions{
RemoteURL: remoteAddr,
Name: form.RepoName,
CloneAddr: remoteAddr,
RepoName: form.RepoName,
Description: form.Description,
Private: form.Private || setting.Repository.ForcePrivate,
Mirror: form.Mirror,
@ -282,47 +315,19 @@ func MigratePost(ctx *context.Context, form auth.MigrateRepoForm) {
opts.Releases = false
}
repo, err := migrations.MigrateRepository(ctx.User, ctxUser.Name, opts)
if err == nil {
notification.NotifyCreateRepository(ctx.User, ctxUser, repo)
log.Trace("Repository migrated [%d]: %s/%s successfully", repo.ID, ctxUser.Name, form.RepoName)
ctx.Redirect(setting.AppSubURL + "/" + ctxUser.Name + "/" + form.RepoName)
err = models.CheckCreateRepository(ctx.User, ctxUser, opts.RepoName)
if err != nil {
handleMigrateError(ctx, ctxUser, err, "MigratePost", tplMigrate, &form)
return
}
switch {
case models.IsErrReachLimitOfRepo(err):
ctx.RenderWithErr(ctx.Tr("repo.form.reach_limit_of_creation", ctxUser.MaxCreationLimit()), tplMigrate, &form)
case models.IsErrNameReserved(err):
ctx.Data["Err_RepoName"] = true
ctx.RenderWithErr(ctx.Tr("repo.form.name_reserved", err.(models.ErrNameReserved).Name), tplMigrate, &form)
case models.IsErrRepoAlreadyExist(err):
ctx.Data["Err_RepoName"] = true
ctx.RenderWithErr(ctx.Tr("form.repo_name_been_taken"), tplMigrate, &form)
case models.IsErrNamePatternNotAllowed(err):
ctx.Data["Err_RepoName"] = true
ctx.RenderWithErr(ctx.Tr("repo.form.name_pattern_not_allowed", err.(models.ErrNamePatternNotAllowed).Pattern), tplMigrate, &form)
case migrations.IsRateLimitError(err):
ctx.RenderWithErr(ctx.Tr("form.visit_rate_limit"), tplMigrate, &form)
case migrations.IsTwoFactorAuthError(err):
ctx.Data["Err_Auth"] = true
ctx.RenderWithErr(ctx.Tr("form.2fa_auth_required"), tplMigrate, &form)
default:
// remoteAddr may contain credentials, so we sanitize it
err = util.URLSanitizedError(err, remoteAddr)
if strings.Contains(err.Error(), "Authentication failed") ||
strings.Contains(err.Error(), "Bad credentials") ||
strings.Contains(err.Error(), "could not read Username") {
ctx.Data["Err_Auth"] = true
ctx.RenderWithErr(ctx.Tr("form.auth_failed", err.Error()), tplMigrate, &form)
} else if strings.Contains(err.Error(), "fatal:") {
ctx.Data["Err_CloneAddr"] = true
ctx.RenderWithErr(ctx.Tr("repo.migrate.failed", err.Error()), tplMigrate, &form)
} else {
ctx.ServerError("MigratePost", err)
}
err = task.MigrateRepository(ctx.User, ctxUser, opts)
if err == nil {
ctx.Redirect(setting.AppSubURL + "/" + ctxUser.Name + "/" + opts.RepoName)
return
}
handleMigrateError(ctx, ctxUser, err, "MigratePost", tplMigrate, &form)
}
// Action response for actions to a repository
@ -460,3 +465,19 @@ func Download(ctx *context.Context) {
ctx.ServeFile(archivePath, ctx.Repo.Repository.Name+"-"+refName+ext)
}
// Status returns repository's status
func Status(ctx *context.Context) {
task, err := models.GetMigratingTask(ctx.Repo.Repository.ID)
if err != nil {
ctx.JSON(500, map[string]interface{}{
"err": err,
})
return
}
ctx.JSON(200, map[string]interface{}{
"status": ctx.Repo.Repository.Status,
"err": task.Errors,
})
}

30
routers/repo/view.go
View file

@ -11,6 +11,7 @@ import (
"fmt"
gotemplate "html/template"
"io/ioutil"
"net/url"
"path"
"strings"
@ -31,6 +32,7 @@ const (
tplRepoHome base.TplName = "repo/home"
tplWatchers base.TplName = "repo/watchers"
tplForks base.TplName = "repo/forks"
tplMigrating base.TplName = "repo/migrating"
)
func renderDirectory(ctx *context.Context, treeLink string) {
@ -356,9 +358,37 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st
}
}
func safeURL(address string) string {
u, err := url.Parse(address)
if err != nil {
return address
}
u.User = nil
return u.String()
}
// Home render repository home page
func Home(ctx *context.Context) {
if len(ctx.Repo.Units) > 0 {
if ctx.Repo.Repository.IsBeingCreated() {
task, err := models.GetMigratingTask(ctx.Repo.Repository.ID)
if err != nil {
ctx.ServerError("models.GetMigratingTask", err)
return
}
cfg, err := task.MigrateConfig()
if err != nil {
ctx.ServerError("task.MigrateConfig", err)
return
}
ctx.Data["Repo"] = ctx.Repo
ctx.Data["MigrateTask"] = task
ctx.Data["CloneAddr"] = safeURL(cfg.CloneAddr)
ctx.HTML(200, tplMigrating)
return
}
var firstUnit *models.Unit
for _, repoUnit := range ctx.Repo.Units {
if repoUnit.Type == models.UnitTypeCode {

2
routers/routes/routes.go
View file

@ -845,6 +845,8 @@ func RegisterRoutes(m *macaron.Macaron) {
m.Get("/archive/*", repo.MustBeNotEmpty, reqRepoCodeReader, repo.Download)
m.Get("/status", reqRepoCodeReader, repo.Status)
m.Group("/branches", func() {
m.Get("", repo.Branches)
}, repo.MustBeNotEmpty, context.RepoRef(), reqRepoCodeReader)