feat: add webhooks manager and switch to using dedicated pull_request webhook endpoint. Also add iniyial pull_request support

api/app.go

@@ -1,11 +1,14 @@
 package api
 
 import (
+	"crypto/hmac"
 	"crypto/rand"
 	"crypto/rsa"
+	"crypto/sha256"
 	"crypto/x509"
 	"encoding/hex"
 	"encoding/pem"
+	"fmt"
 	"git.lumeweb.com/LumeWeb/gitea-github-proxy/config"
 	"go.uber.org/zap"
 	"gorm.io/gorm"
@@ -70,3 +73,10 @@ func generateTempCode() string {
 	}
 	return hex.EncodeToString(bytes)
 }
+
+func generatePayloadSignature(payload []byte, secret string) string {
+	hasher := hmac.New(sha256.New, []byte(secret))
+	hasher.Write(payload)
+	sha := hex.EncodeToString(hasher.Sum(nil))
+	return fmt.Sprintf("sha256=%s", sha)
+}

api/middleware.go

@@ -3,15 +3,18 @@ package api
 import (
 	"code.gitea.io/sdk/gitea"
 	"context"
+	"encoding/json"
 	"git.lumeweb.com/LumeWeb/gitea-github-proxy/config"
 	"github.com/gorilla/mux"
 	"go.uber.org/zap"
+	"io"
 	"net/http"
 	"strings"
 )
 
 const AUTHED_CONTEXT_KEY = "authed"
 const REDIRECT_AFTER_AUTH = "redirect-after-auth"
+const WEBHOOK_CONTEXT_KEY = "webhook"
 
 const AuthCookieName = "auth-token"
 
@@ -60,6 +63,40 @@ func giteaOauthVerifyMiddleware(cfg *config.Config) mux.MiddlewareFunc {
 	}
 }
 
+func verifyWebhookDataMiddleware(cfg *config.Config, logger *zap.Logger) mux.MiddlewareFunc {
+	return func(next http.Handler) http.Handler {
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			var webhook map[string]interface{}
+			body, err := io.ReadAll(r.Body)
+			if err != nil {
+				logger.Error("Failed to read request body", zap.Error(err))
+				w.WriteHeader(http.StatusInternalServerError)
+				return
+			}
+
+			err = json.Unmarshal(body, &webhook)
+			if err != nil {
+				logger.Error("Failed to unmarshal webhook", zap.Error(err))
+				w.WriteHeader(http.StatusInternalServerError)
+				return
+			}
+
+			if len(webhook) == 0 {
+				logger.Error("Webhook data is empty")
+				w.WriteHeader(http.StatusBadRequest)
+				return
+			}
+
+			ctx := context.WithValue(r.Context(), WEBHOOK_CONTEXT_KEY, body)
+			r = r.WithContext(ctx)
+
+			r.Body = io.NopCloser(strings.NewReader(string(body)))
+
+			next.ServeHTTP(w, r)
+		})
+	}
+}
+
 func requireAuthMiddleware(cfg *config.Config) mux.MiddlewareFunc {
 	return func(next http.Handler) http.Handler {
 		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

api/routes.go

@@ -11,10 +11,11 @@ import (
 
 type RouteParams struct {
 	fx.In
-	Config *config.Config
+	Config         *config.Config
-	Db     *gorm.DB
+	Db             *gorm.DB
-	Logger *zap.Logger
+	Logger         *zap.Logger
-	R      *mux.Router
+	R              *mux.Router
+	WebhookManager *WebhookManager
 }
 
 func SetupRoutes(params RouteParams) {

api/routes_webhooks.go

@@ -1,12 +1,12 @@
 package api
 
 import (
+	"code.gitea.io/gitea/modules/structs"
 	"code.gitea.io/sdk/gitea"
 	"encoding/json"
 	"git.lumeweb.com/LumeWeb/gitea-github-proxy/config"
 	"go.uber.org/zap"
 	"gorm.io/gorm"
-	"io"
 	"net/http"
 )
 
@@ -14,24 +14,23 @@ type webhookApi struct {
 	config *config.Config
 	logger *zap.Logger
 	db     *gorm.DB
+	whm    *WebhookManager
 }
 
-func newWebhookApi(cfg *config.Config, db *gorm.DB, logger *zap.Logger) *webhookApi {
+func newWebhookApi(cfg *config.Config, db *gorm.DB, logger *zap.Logger, whm *WebhookManager) *webhookApi {
-	return &webhookApi{config: cfg, db: db, logger: logger}
+	return &webhookApi{config: cfg, db: db, logger: logger, whm: whm}
 }
 
-func (a *webhookApi) handleWebhook(w http.ResponseWriter, r *http.Request) {
+func (a *webhookApi) handlePullRequest(w http.ResponseWriter, r *http.Request) {
-	body, _ := io.ReadAll(r.Body)
+	webhook := a.getWebhookData(w, r)
 
-	var webhook map[string]interface{}
+	payload := &structs.PullRequestPayload{}
-	err := json.Unmarshal(body, &webhook)
+
-	if err != nil {
+	err := json.Unmarshal(webhook, payload)
-		a.logger.Error("Failed to unmarshal webhook", zap.Error(err))
+	if a.unmarshallError(w, err) {
-		w.WriteHeader(http.StatusInternalServerError)
 		return
 	}
 
-	a.logger.Debug("Received webhook", zap.Any("webhook", webhook))
 }
 
 func setupWebhookRoutes(params RouteParams) {
@@ -39,11 +38,34 @@ func setupWebhookRoutes(params RouteParams) {
 	db := params.Db
 	logger := params.Logger
 	r := params.R
+	whm := params.WebhookManager
 
-	webhookApi := newWebhookApi(cfg, db, logger)
+	webhookApi := newWebhookApi(cfg, db, logger, whm)
 	webhookRouter := r.PathPrefix("/api").Subrouter()
 
 	webhookRouter.Use(gitea.VerifyWebhookSignatureMiddleware(cfg.GiteaWebHookSecret))
 
-	webhookRouter.HandleFunc("/webhook", webhookApi.handleWebhook).Methods("POST")
+	webhookRouter.HandleFunc("/webhooks/pull_request", webhookApi.handlePullRequest).Methods("POST")
+}
+
+func (a *webhookApi) getWebhookData(w http.ResponseWriter, r *http.Request) []byte {
+	return getWebhookData(r, a.logger)
+}
+
+func (a *webhookApi) unmarshallError(w http.ResponseWriter, err error) bool {
+	if err != nil {
+		a.logger.Error("Failed to unmarshal webhook data", zap.Error(err))
+		http.Error(w, "Failed to unmarshal webhook data", http.StatusInternalServerError)
+	}
+
+	return err != nil
+}
+func getWebhookData(r *http.Request, logger *zap.Logger) []byte {
+	webhook := r.Context().Value(WEBHOOK_CONTEXT_KEY)
+	webhookData, ok := webhook.([]byte)
+	if !ok {
+		logger.Fatal("Failed to get webhook data from context")
+	}
+
+	return webhookData
 }

api/webhook_manager.go

@@ -0,0 +1,294 @@
+package api
+
+import (
+	"bytes"
+	"code.gitea.io/gitea/modules/structs"
+	"encoding/json"
+	"git.lumeweb.com/LumeWeb/gitea-github-proxy/config"
+	"git.lumeweb.com/LumeWeb/gitea-github-proxy/db/model"
+	"github.com/google/go-github/v59/github"
+	"go.uber.org/zap"
+	"gorm.io/gorm"
+	"io"
+	"net/http"
+	"time"
+)
+
+type WebhookManager struct {
+	config *config.Config
+	db     *gorm.DB
+	logger *zap.Logger
+}
+
+type IncomingWebhookData struct {
+	Headers http.Header
+}
+
+func NewWebhookManager(cfg *config.Config, db *gorm.DB, logger *zap.Logger) *WebhookManager {
+	return &WebhookManager{config: cfg, db: db, logger: logger}
+}
+
+func (whm *WebhookManager) HandlePullRequest(request *structs.PullRequestPayload, r *http.Request) {
+	// Mapping the sender
+	sender := &github.User{
+		Login:     &request.Sender.UserName,
+		ID:        &request.Sender.ID,
+		AvatarURL: &request.Sender.AvatarURL,
+		Name:      &request.Sender.FullName,
+		Email:     &request.Sender.Email,
+		SiteAdmin: &request.Sender.IsAdmin,
+	}
+	repo := &github.Repository{
+		ID:               int64Ptr(request.Repository.ID),
+		Name:             stringPtr(request.Repository.Name),
+		FullName:         stringPtr(request.Repository.FullName),
+		Description:      stringPtr(request.Repository.Description),
+		Homepage:         stringPtr(request.Repository.Website),
+		HTMLURL:          stringPtr(request.Repository.HTMLURL),
+		CloneURL:         stringPtr(request.Repository.CloneURL),
+		GitURL:           stringPtr(request.Repository.CloneURL),
+		SSHURL:           stringPtr(request.Repository.SSHURL),
+		DefaultBranch:    stringPtr(request.Repository.DefaultBranch),
+		CreatedAt:        timePtr(request.Repository.Created),
+		UpdatedAt:        timePtr(request.Repository.Updated),
+		Private:          boolPtr(request.Repository.Private),
+		Fork:             boolPtr(request.Repository.Fork),
+		Size:             intPtr(request.Repository.Size),
+		StargazersCount:  intPtr(request.Repository.Stars),
+		SubscribersCount: intPtr(request.Repository.Watchers),
+		ForksCount:       intPtr(request.Repository.Forks),
+		Watchers:         intPtr(request.Repository.Watchers),
+		WatchersCount:    intPtr(request.Repository.Stars),
+		OpenIssuesCount:  intPtr(request.Repository.OpenIssues),
+		Archived:         boolPtr(request.Repository.Archived),
+	}
+
+	pr := &github.PullRequest{
+		ID:             int64Ptr(request.PullRequest.ID),
+		Number:         intPtr(int(request.PullRequest.Index)),
+		State:          stringPtr(string(request.PullRequest.State)),
+		Title:          stringPtr(request.PullRequest.Title),
+		Body:           stringPtr(request.PullRequest.Body),
+		CreatedAt:      timePtr(*request.PullRequest.Created),
+		UpdatedAt:      timePtr(*request.PullRequest.Updated),
+		ClosedAt:       timePtrIfNotNil(request.PullRequest.Closed),
+		MergedAt:       timePtrIfNotNil(request.PullRequest.Merged),
+		Merged:         boolPtr(request.PullRequest.HasMerged),
+		Mergeable:      boolPtr(request.PullRequest.Mergeable),
+		MergeCommitSHA: request.PullRequest.MergedCommitID,
+		URL:            stringPtr(request.PullRequest.URL),
+		HTMLURL:        stringPtr(request.PullRequest.HTMLURL),
+		DiffURL:        stringPtr(request.PullRequest.DiffURL),
+		PatchURL:       stringPtr(request.PullRequest.PatchURL),
+		Comments:       intPtr(request.PullRequest.Comments),
+		Assignee:       convertUser(request.PullRequest.Assignee),
+		Assignees:      convertUsers(request.PullRequest.Assignees),
+		Milestone:      convertMilestone(request.PullRequest.Milestone),
+		Labels:         convertLabels(request.PullRequest.Labels),
+	}
+
+	// Convert PR branch info
+	if request.PullRequest.Head != nil {
+		pr.Head = convertPRBranch(request.PullRequest.Head)
+	}
+	if request.PullRequest.Base != nil {
+		pr.Base = convertPRBranch(request.PullRequest.Base)
+	}
+
+	changes := &github.EditChange{
+		Title: &github.EditTitle{
+			From: stringPtr(request.Changes.Title.From),
+		},
+		Body: &github.EditBody{
+			From: stringPtr(request.Changes.Body.From),
+		},
+		Base: &github.EditBase{
+			Ref: &github.EditRef{
+				From: stringPtr(request.Changes.Ref.From),
+			},
+		},
+	}
+
+	ghEvent := &github.PullRequestEvent{
+		Action:      stringPtr(string(request.Action)),
+		Number:      numberPtr(int(request.Index)),
+		PullRequest: pr,
+		Changes:     changes,
+		Repo:        repo,
+		Sender:      sender,
+	}
+	githubAction := ""
+
+	switch request.Action {
+	case structs.HookIssueOpened:
+		githubAction = "opened"
+	case structs.HookIssueClosed:
+		githubAction = "closed"
+	case structs.HookIssueReOpened:
+		githubAction = "reopened"
+	case structs.HookIssueEdited:
+		githubAction = "edited"
+	case structs.HookIssueAssigned:
+		githubAction = "assigned"
+	case structs.HookIssueUnassigned:
+		githubAction = "unassigned"
+	case structs.HookIssueLabelUpdated:
+		// GitHub does not have a direct "label_updated" event; use "labeled" as the closest action
+		githubAction = "labeled" // Assuming you handle the update as adding a label
+	case structs.HookIssueLabelCleared:
+		// GitHub does not have a direct "label_cleared" event; use "unlabeled" as the closest action
+		githubAction = "unlabeled" // Assuming you handle the clearing as removing a label
+	case structs.HookIssueSynchronized:
+		githubAction = "synchronize"
+	case structs.HookIssueMilestoned:
+		githubAction = "milestoned"
+	case structs.HookIssueDemilestoned:
+		githubAction = "demilestoned"
+	case structs.HookIssueReviewed:
+		// GitHub does not have a direct "reviewed" event for PRs; this might be closest to a review submitted
+		githubAction = "review_submitted" // This is not a direct GitHub event, consider how best to map this action
+	case structs.HookIssueReviewRequested:
+		githubAction = "review_requested"
+	case structs.HookIssueReviewRequestRemoved:
+		githubAction = "review_request_removed"
+	default:
+		// Fallback for any unhandled actions
+		githubAction = "unknown_action"
+	}
+	r.Header.Set("X-GitHub-Event", githubAction)
+
+	whm.sendWebhooks(ghEvent, r)
+}
+
+func stringPtr(s string) *string {
+	return &s
+}
+
+func numberPtr(n int) *int {
+	return &n
+}
+
+func intPtr(n int) *int {
+	return &n
+}
+
+func int64Ptr(n int64) *int64 {
+	return &n
+}
+
+func boolPtr(b bool) *bool {
+	return &b
+}
+
+func timePtr(t time.Time) *github.Timestamp {
+	timestamp := github.Timestamp{Time: t}
+	return &timestamp
+}
+
+func timePtrIfNotNil(t *time.Time) *github.Timestamp {
+	if t == nil {
+		return nil
+	}
+	return timePtr(*t)
+}
+
+func convertUser(user *structs.User) *github.User {
+	if user == nil {
+		return nil
+	}
+	return &github.User{
+		Login: stringPtr(user.UserName),
+		ID:    int64Ptr(user.ID),
+	}
+}
+
+func convertUsers(users []*structs.User) []*github.User {
+	var ghUsers []*github.User
+	for _, user := range users {
+		ghUsers = append(ghUsers, convertUser(user))
+	}
+	return ghUsers
+}
+
+func convertMilestone(milestone *structs.Milestone) *github.Milestone {
+	if milestone == nil {
+		return nil
+	}
+	return &github.Milestone{
+		Title: stringPtr(milestone.Title),
+	}
+}
+
+func convertLabels(labels []*structs.Label) []*github.Label {
+	var ghLabels []*github.Label
+	for _, label := range labels {
+		ghLabels = append(ghLabels, &github.Label{
+			Name: stringPtr(label.Name),
+		})
+	}
+	return ghLabels
+}
+
+func convertPRBranch(branch *structs.PRBranchInfo) *github.PullRequestBranch {
+	return &github.PullRequestBranch{
+		Label: stringPtr(branch.Name),
+		Ref:   stringPtr(branch.Ref),
+		SHA:   stringPtr(branch.Sha),
+	}
+}
+
+func (whm *WebhookManager) sendWebhooks(request interface{}, r *http.Request) {
+	var apps []model.Apps
+	result := whm.db.Find(&apps)
+	if result.Error != nil {
+		whm.logger.Error("Failed to query apps", zap.Error(result.Error))
+		return
+	}
+	for _, app := range apps {
+		go func(app model.Apps) {
+			payloadBytes, err := json.Marshal(request)
+			if err != nil {
+				whm.logger.Error("Failed to marshal payload", zap.Error(err))
+				return
+			}
+
+			req, err := http.NewRequest("POST", app.WebhookUrl, bytes.NewBuffer(payloadBytes))
+			if err != nil {
+				whm.logger.Error("Failed to create request", zap.Error(err), zap.String("url", app.WebhookUrl))
+				return
+			}
+
+			req.Header.Set("Content-Type", "application/json")
+
+			for key, values := range r.Header {
+				for _, value := range values {
+					req.Header.Add(key, value)
+				}
+			}
+
+			payload := getWebhookData(r, whm.logger)
+			signature := generatePayloadSignature(payload, app.WebhookSecret)
+			req.Header.Add("X-Hub-Signature-256", signature)
+
+			client := &http.Client{}
+			resp, err := client.Do(req)
+			if err != nil {
+				whm.logger.Error("Failed to send webhook", zap.Error(err), zap.String("url", app.WebhookUrl))
+				return
+			}
+			defer func(Body io.ReadCloser) {
+				err := Body.Close()
+				if err != nil {
+					whm.logger.Error("Failed to close response body", zap.Error(err))
+				}
+			}(resp.Body)
+
+			if resp.StatusCode >= 200 && resp.StatusCode < 300 {
+				whm.logger.Info("Webhook sent successfully", zap.String("url", app.WebhookUrl))
+			} else {
+				whm.logger.Error("Webhook failed", zap.String("url", app.WebhookUrl), zap.Int("status", resp.StatusCode))
+			}
+		}(app)
+	}
+
+}

cmd/proxy/main.go

@@ -26,6 +26,7 @@ func main() {
 		db.Module,
 		fx.Provide(api.NewRouter),
 		fx.Provide(NewServer),
+		fx.Provide(api.NewWebhookManager),
 		fx.Invoke(api.SetupRoutes),
 		fx.Invoke(func(*http.Server) {}),
 	).Run()

go.mod

@@ -3,13 +3,15 @@ module git.lumeweb.com/LumeWeb/gitea-github-proxy
 go 1.21
 
 require (
-	code.gitea.io/gitea/modules/structs v0.0.0-20190610152049-835b53fc259c
+	code.gitea.io/gitea v1.21.5
 	code.gitea.io/sdk/gitea v0.17.1
+	github.com/golang-jwt/jwt v3.2.2+incompatible
+	github.com/google/go-github/v59 v59.0.0
 	github.com/gorilla/mux v1.8.1
 	github.com/mitchellh/mapstructure v1.5.0
 	github.com/spf13/viper v1.18.2
 	go.uber.org/fx v1.20.1
-	go.uber.org/zap v1.23.0
+	go.uber.org/zap v1.26.0
 	golang.org/x/oauth2 v0.17.0
 	gorm.io/driver/sqlite v1.5.5
 	gorm.io/gorm v1.25.7
@@ -18,14 +20,18 @@ require (
 require (
 	github.com/davidmz/go-pageant v1.0.2 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
-	github.com/go-fed/httpsig v1.1.0 // indirect
+	github.com/go-fed/httpsig v1.1.1-0.20201223112313-55836744818e // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/hashicorp/go-version v1.6.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/jinzhu/now v1.1.5 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mattn/go-sqlite3 v1.14.17 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/pelletier/go-toml/v2 v2.1.0 // indirect
 	github.com/sagikazarmark/locafero v0.4.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
@@ -34,15 +40,13 @@ require (
 	github.com/spf13/cast v1.6.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
-	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/dig v1.17.0 // indirect
-	go.uber.org/multierr v1.9.0 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/crypto v0.19.0 // indirect
-	golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
+	golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
-	golang.org/x/net v0.21.0 // indirect
 	golang.org/x/sys v0.17.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
-	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/appengine v1.6.8 // indirect
 	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect