From 36a661bd7136da5cdc788c700b22dcb197e75aa3 Mon Sep 17 00:00:00 2001 From: Derrick Hammer Date: Sun, 11 Feb 2024 03:56:32 -0500 Subject: [PATCH] feat: add webhooks manager and switch to using dedicated pull_request webhook endpoint. Also add iniyial pull_request support --- api/app.go | 10 ++ api/middleware.go | 37 ++++++ api/routes.go | 9 +- api/routes_webhooks.go | 48 +++++-- api/webhook_manager.go | 294 +++++++++++++++++++++++++++++++++++++++++ cmd/proxy/main.go | 1 + go.mod | 20 +-- 7 files changed, 394 insertions(+), 25 deletions(-) create mode 100644 api/webhook_manager.go diff --git a/api/app.go b/api/app.go index 6ff0f59..b1095c9 100644 --- a/api/app.go +++ b/api/app.go @@ -1,11 +1,14 @@ package api import ( + "crypto/hmac" "crypto/rand" "crypto/rsa" + "crypto/sha256" "crypto/x509" "encoding/hex" "encoding/pem" + "fmt" "git.lumeweb.com/LumeWeb/gitea-github-proxy/config" "go.uber.org/zap" "gorm.io/gorm" @@ -70,3 +73,10 @@ func generateTempCode() string { } return hex.EncodeToString(bytes) } + +func generatePayloadSignature(payload []byte, secret string) string { + hasher := hmac.New(sha256.New, []byte(secret)) + hasher.Write(payload) + sha := hex.EncodeToString(hasher.Sum(nil)) + return fmt.Sprintf("sha256=%s", sha) +} diff --git a/api/middleware.go b/api/middleware.go index c61d10c..8093970 100644 --- a/api/middleware.go +++ b/api/middleware.go @@ -3,15 +3,18 @@ package api import ( "code.gitea.io/sdk/gitea" "context" + "encoding/json" "git.lumeweb.com/LumeWeb/gitea-github-proxy/config" "github.com/gorilla/mux" "go.uber.org/zap" + "io" "net/http" "strings" ) const AUTHED_CONTEXT_KEY = "authed" const REDIRECT_AFTER_AUTH = "redirect-after-auth" +const WEBHOOK_CONTEXT_KEY = "webhook" const AuthCookieName = "auth-token" @@ -60,6 +63,40 @@ func giteaOauthVerifyMiddleware(cfg *config.Config) mux.MiddlewareFunc { } } +func verifyWebhookDataMiddleware(cfg *config.Config, logger *zap.Logger) mux.MiddlewareFunc { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var webhook map[string]interface{} + body, err := io.ReadAll(r.Body) + if err != nil { + logger.Error("Failed to read request body", zap.Error(err)) + w.WriteHeader(http.StatusInternalServerError) + return + } + + err = json.Unmarshal(body, &webhook) + if err != nil { + logger.Error("Failed to unmarshal webhook", zap.Error(err)) + w.WriteHeader(http.StatusInternalServerError) + return + } + + if len(webhook) == 0 { + logger.Error("Webhook data is empty") + w.WriteHeader(http.StatusBadRequest) + return + } + + ctx := context.WithValue(r.Context(), WEBHOOK_CONTEXT_KEY, body) + r = r.WithContext(ctx) + + r.Body = io.NopCloser(strings.NewReader(string(body))) + + next.ServeHTTP(w, r) + }) + } +} + func requireAuthMiddleware(cfg *config.Config) mux.MiddlewareFunc { return func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/api/routes.go b/api/routes.go index 9d9cab5..0469813 100644 --- a/api/routes.go +++ b/api/routes.go @@ -11,10 +11,11 @@ import ( type RouteParams struct { fx.In - Config *config.Config - Db *gorm.DB - Logger *zap.Logger - R *mux.Router + Config *config.Config + Db *gorm.DB + Logger *zap.Logger + R *mux.Router + WebhookManager *WebhookManager } func SetupRoutes(params RouteParams) { diff --git a/api/routes_webhooks.go b/api/routes_webhooks.go index f78ed33..dd71ec2 100644 --- a/api/routes_webhooks.go +++ b/api/routes_webhooks.go @@ -1,12 +1,12 @@ package api import ( + "code.gitea.io/gitea/modules/structs" "code.gitea.io/sdk/gitea" "encoding/json" "git.lumeweb.com/LumeWeb/gitea-github-proxy/config" "go.uber.org/zap" "gorm.io/gorm" - "io" "net/http" ) @@ -14,24 +14,23 @@ type webhookApi struct { config *config.Config logger *zap.Logger db *gorm.DB + whm *WebhookManager } -func newWebhookApi(cfg *config.Config, db *gorm.DB, logger *zap.Logger) *webhookApi { - return &webhookApi{config: cfg, db: db, logger: logger} +func newWebhookApi(cfg *config.Config, db *gorm.DB, logger *zap.Logger, whm *WebhookManager) *webhookApi { + return &webhookApi{config: cfg, db: db, logger: logger, whm: whm} } -func (a *webhookApi) handleWebhook(w http.ResponseWriter, r *http.Request) { - body, _ := io.ReadAll(r.Body) +func (a *webhookApi) handlePullRequest(w http.ResponseWriter, r *http.Request) { + webhook := a.getWebhookData(w, r) - var webhook map[string]interface{} - err := json.Unmarshal(body, &webhook) - if err != nil { - a.logger.Error("Failed to unmarshal webhook", zap.Error(err)) - w.WriteHeader(http.StatusInternalServerError) + payload := &structs.PullRequestPayload{} + + err := json.Unmarshal(webhook, payload) + if a.unmarshallError(w, err) { return } - a.logger.Debug("Received webhook", zap.Any("webhook", webhook)) } func setupWebhookRoutes(params RouteParams) { @@ -39,11 +38,34 @@ func setupWebhookRoutes(params RouteParams) { db := params.Db logger := params.Logger r := params.R + whm := params.WebhookManager - webhookApi := newWebhookApi(cfg, db, logger) + webhookApi := newWebhookApi(cfg, db, logger, whm) webhookRouter := r.PathPrefix("/api").Subrouter() webhookRouter.Use(gitea.VerifyWebhookSignatureMiddleware(cfg.GiteaWebHookSecret)) - webhookRouter.HandleFunc("/webhook", webhookApi.handleWebhook).Methods("POST") + webhookRouter.HandleFunc("/webhooks/pull_request", webhookApi.handlePullRequest).Methods("POST") +} + +func (a *webhookApi) getWebhookData(w http.ResponseWriter, r *http.Request) []byte { + return getWebhookData(r, a.logger) +} + +func (a *webhookApi) unmarshallError(w http.ResponseWriter, err error) bool { + if err != nil { + a.logger.Error("Failed to unmarshal webhook data", zap.Error(err)) + http.Error(w, "Failed to unmarshal webhook data", http.StatusInternalServerError) + } + + return err != nil +} +func getWebhookData(r *http.Request, logger *zap.Logger) []byte { + webhook := r.Context().Value(WEBHOOK_CONTEXT_KEY) + webhookData, ok := webhook.([]byte) + if !ok { + logger.Fatal("Failed to get webhook data from context") + } + + return webhookData } diff --git a/api/webhook_manager.go b/api/webhook_manager.go new file mode 100644 index 0000000..5496f56 --- /dev/null +++ b/api/webhook_manager.go @@ -0,0 +1,294 @@ +package api + +import ( + "bytes" + "code.gitea.io/gitea/modules/structs" + "encoding/json" + "git.lumeweb.com/LumeWeb/gitea-github-proxy/config" + "git.lumeweb.com/LumeWeb/gitea-github-proxy/db/model" + "github.com/google/go-github/v59/github" + "go.uber.org/zap" + "gorm.io/gorm" + "io" + "net/http" + "time" +) + +type WebhookManager struct { + config *config.Config + db *gorm.DB + logger *zap.Logger +} + +type IncomingWebhookData struct { + Headers http.Header +} + +func NewWebhookManager(cfg *config.Config, db *gorm.DB, logger *zap.Logger) *WebhookManager { + return &WebhookManager{config: cfg, db: db, logger: logger} +} + +func (whm *WebhookManager) HandlePullRequest(request *structs.PullRequestPayload, r *http.Request) { + // Mapping the sender + sender := &github.User{ + Login: &request.Sender.UserName, + ID: &request.Sender.ID, + AvatarURL: &request.Sender.AvatarURL, + Name: &request.Sender.FullName, + Email: &request.Sender.Email, + SiteAdmin: &request.Sender.IsAdmin, + } + repo := &github.Repository{ + ID: int64Ptr(request.Repository.ID), + Name: stringPtr(request.Repository.Name), + FullName: stringPtr(request.Repository.FullName), + Description: stringPtr(request.Repository.Description), + Homepage: stringPtr(request.Repository.Website), + HTMLURL: stringPtr(request.Repository.HTMLURL), + CloneURL: stringPtr(request.Repository.CloneURL), + GitURL: stringPtr(request.Repository.CloneURL), + SSHURL: stringPtr(request.Repository.SSHURL), + DefaultBranch: stringPtr(request.Repository.DefaultBranch), + CreatedAt: timePtr(request.Repository.Created), + UpdatedAt: timePtr(request.Repository.Updated), + Private: boolPtr(request.Repository.Private), + Fork: boolPtr(request.Repository.Fork), + Size: intPtr(request.Repository.Size), + StargazersCount: intPtr(request.Repository.Stars), + SubscribersCount: intPtr(request.Repository.Watchers), + ForksCount: intPtr(request.Repository.Forks), + Watchers: intPtr(request.Repository.Watchers), + WatchersCount: intPtr(request.Repository.Stars), + OpenIssuesCount: intPtr(request.Repository.OpenIssues), + Archived: boolPtr(request.Repository.Archived), + } + + pr := &github.PullRequest{ + ID: int64Ptr(request.PullRequest.ID), + Number: intPtr(int(request.PullRequest.Index)), + State: stringPtr(string(request.PullRequest.State)), + Title: stringPtr(request.PullRequest.Title), + Body: stringPtr(request.PullRequest.Body), + CreatedAt: timePtr(*request.PullRequest.Created), + UpdatedAt: timePtr(*request.PullRequest.Updated), + ClosedAt: timePtrIfNotNil(request.PullRequest.Closed), + MergedAt: timePtrIfNotNil(request.PullRequest.Merged), + Merged: boolPtr(request.PullRequest.HasMerged), + Mergeable: boolPtr(request.PullRequest.Mergeable), + MergeCommitSHA: request.PullRequest.MergedCommitID, + URL: stringPtr(request.PullRequest.URL), + HTMLURL: stringPtr(request.PullRequest.HTMLURL), + DiffURL: stringPtr(request.PullRequest.DiffURL), + PatchURL: stringPtr(request.PullRequest.PatchURL), + Comments: intPtr(request.PullRequest.Comments), + Assignee: convertUser(request.PullRequest.Assignee), + Assignees: convertUsers(request.PullRequest.Assignees), + Milestone: convertMilestone(request.PullRequest.Milestone), + Labels: convertLabels(request.PullRequest.Labels), + } + + // Convert PR branch info + if request.PullRequest.Head != nil { + pr.Head = convertPRBranch(request.PullRequest.Head) + } + if request.PullRequest.Base != nil { + pr.Base = convertPRBranch(request.PullRequest.Base) + } + + changes := &github.EditChange{ + Title: &github.EditTitle{ + From: stringPtr(request.Changes.Title.From), + }, + Body: &github.EditBody{ + From: stringPtr(request.Changes.Body.From), + }, + Base: &github.EditBase{ + Ref: &github.EditRef{ + From: stringPtr(request.Changes.Ref.From), + }, + }, + } + + ghEvent := &github.PullRequestEvent{ + Action: stringPtr(string(request.Action)), + Number: numberPtr(int(request.Index)), + PullRequest: pr, + Changes: changes, + Repo: repo, + Sender: sender, + } + githubAction := "" + + switch request.Action { + case structs.HookIssueOpened: + githubAction = "opened" + case structs.HookIssueClosed: + githubAction = "closed" + case structs.HookIssueReOpened: + githubAction = "reopened" + case structs.HookIssueEdited: + githubAction = "edited" + case structs.HookIssueAssigned: + githubAction = "assigned" + case structs.HookIssueUnassigned: + githubAction = "unassigned" + case structs.HookIssueLabelUpdated: + // GitHub does not have a direct "label_updated" event; use "labeled" as the closest action + githubAction = "labeled" // Assuming you handle the update as adding a label + case structs.HookIssueLabelCleared: + // GitHub does not have a direct "label_cleared" event; use "unlabeled" as the closest action + githubAction = "unlabeled" // Assuming you handle the clearing as removing a label + case structs.HookIssueSynchronized: + githubAction = "synchronize" + case structs.HookIssueMilestoned: + githubAction = "milestoned" + case structs.HookIssueDemilestoned: + githubAction = "demilestoned" + case structs.HookIssueReviewed: + // GitHub does not have a direct "reviewed" event for PRs; this might be closest to a review submitted + githubAction = "review_submitted" // This is not a direct GitHub event, consider how best to map this action + case structs.HookIssueReviewRequested: + githubAction = "review_requested" + case structs.HookIssueReviewRequestRemoved: + githubAction = "review_request_removed" + default: + // Fallback for any unhandled actions + githubAction = "unknown_action" + } + r.Header.Set("X-GitHub-Event", githubAction) + + whm.sendWebhooks(ghEvent, r) +} + +func stringPtr(s string) *string { + return &s +} + +func numberPtr(n int) *int { + return &n +} + +func intPtr(n int) *int { + return &n +} + +func int64Ptr(n int64) *int64 { + return &n +} + +func boolPtr(b bool) *bool { + return &b +} + +func timePtr(t time.Time) *github.Timestamp { + timestamp := github.Timestamp{Time: t} + return ×tamp +} + +func timePtrIfNotNil(t *time.Time) *github.Timestamp { + if t == nil { + return nil + } + return timePtr(*t) +} + +func convertUser(user *structs.User) *github.User { + if user == nil { + return nil + } + return &github.User{ + Login: stringPtr(user.UserName), + ID: int64Ptr(user.ID), + } +} + +func convertUsers(users []*structs.User) []*github.User { + var ghUsers []*github.User + for _, user := range users { + ghUsers = append(ghUsers, convertUser(user)) + } + return ghUsers +} + +func convertMilestone(milestone *structs.Milestone) *github.Milestone { + if milestone == nil { + return nil + } + return &github.Milestone{ + Title: stringPtr(milestone.Title), + } +} + +func convertLabels(labels []*structs.Label) []*github.Label { + var ghLabels []*github.Label + for _, label := range labels { + ghLabels = append(ghLabels, &github.Label{ + Name: stringPtr(label.Name), + }) + } + return ghLabels +} + +func convertPRBranch(branch *structs.PRBranchInfo) *github.PullRequestBranch { + return &github.PullRequestBranch{ + Label: stringPtr(branch.Name), + Ref: stringPtr(branch.Ref), + SHA: stringPtr(branch.Sha), + } +} + +func (whm *WebhookManager) sendWebhooks(request interface{}, r *http.Request) { + var apps []model.Apps + result := whm.db.Find(&apps) + if result.Error != nil { + whm.logger.Error("Failed to query apps", zap.Error(result.Error)) + return + } + for _, app := range apps { + go func(app model.Apps) { + payloadBytes, err := json.Marshal(request) + if err != nil { + whm.logger.Error("Failed to marshal payload", zap.Error(err)) + return + } + + req, err := http.NewRequest("POST", app.WebhookUrl, bytes.NewBuffer(payloadBytes)) + if err != nil { + whm.logger.Error("Failed to create request", zap.Error(err), zap.String("url", app.WebhookUrl)) + return + } + + req.Header.Set("Content-Type", "application/json") + + for key, values := range r.Header { + for _, value := range values { + req.Header.Add(key, value) + } + } + + payload := getWebhookData(r, whm.logger) + signature := generatePayloadSignature(payload, app.WebhookSecret) + req.Header.Add("X-Hub-Signature-256", signature) + + client := &http.Client{} + resp, err := client.Do(req) + if err != nil { + whm.logger.Error("Failed to send webhook", zap.Error(err), zap.String("url", app.WebhookUrl)) + return + } + defer func(Body io.ReadCloser) { + err := Body.Close() + if err != nil { + whm.logger.Error("Failed to close response body", zap.Error(err)) + } + }(resp.Body) + + if resp.StatusCode >= 200 && resp.StatusCode < 300 { + whm.logger.Info("Webhook sent successfully", zap.String("url", app.WebhookUrl)) + } else { + whm.logger.Error("Webhook failed", zap.String("url", app.WebhookUrl), zap.Int("status", resp.StatusCode)) + } + }(app) + } + +} diff --git a/cmd/proxy/main.go b/cmd/proxy/main.go index 70bd548..d62fb5b 100644 --- a/cmd/proxy/main.go +++ b/cmd/proxy/main.go @@ -26,6 +26,7 @@ func main() { db.Module, fx.Provide(api.NewRouter), fx.Provide(NewServer), + fx.Provide(api.NewWebhookManager), fx.Invoke(api.SetupRoutes), fx.Invoke(func(*http.Server) {}), ).Run() diff --git a/go.mod b/go.mod index 9235701..8c40bbf 100644 --- a/go.mod +++ b/go.mod @@ -3,13 +3,15 @@ module git.lumeweb.com/LumeWeb/gitea-github-proxy go 1.21 require ( - code.gitea.io/gitea/modules/structs v0.0.0-20190610152049-835b53fc259c + code.gitea.io/gitea v1.21.5 code.gitea.io/sdk/gitea v0.17.1 + github.com/golang-jwt/jwt v3.2.2+incompatible + github.com/google/go-github/v59 v59.0.0 github.com/gorilla/mux v1.8.1 github.com/mitchellh/mapstructure v1.5.0 github.com/spf13/viper v1.18.2 go.uber.org/fx v1.20.1 - go.uber.org/zap v1.23.0 + go.uber.org/zap v1.26.0 golang.org/x/oauth2 v0.17.0 gorm.io/driver/sqlite v1.5.5 gorm.io/gorm v1.25.7 @@ -18,14 +20,18 @@ require ( require ( github.com/davidmz/go-pageant v1.0.2 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect - github.com/go-fed/httpsig v1.1.0 // indirect + github.com/go-fed/httpsig v1.1.1-0.20201223112313-55836744818e // indirect github.com/golang/protobuf v1.5.3 // indirect + github.com/google/go-querystring v1.1.0 // indirect github.com/hashicorp/go-version v1.6.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect + github.com/json-iterator/go v1.1.12 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-sqlite3 v1.14.17 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml/v2 v2.1.0 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect @@ -34,15 +40,13 @@ require ( github.com/spf13/cast v1.6.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/subosito/gotenv v1.6.0 // indirect - go.uber.org/atomic v1.9.0 // indirect go.uber.org/dig v1.17.0 // indirect - go.uber.org/multierr v1.9.0 // indirect + go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.19.0 // indirect - golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect - golang.org/x/net v0.21.0 // indirect + golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect golang.org/x/sys v0.17.0 // indirect golang.org/x/text v0.14.0 // indirect - google.golang.org/appengine v1.6.7 // indirect + google.golang.org/appengine v1.6.8 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect