Skip to content

Commit 60d57f4

Browse files
olblakolblak
authored
feat: Allow to use GitHub search to target multipe GitHub repositories (#6422)
* refactor GitHub client Signed-off-by: Olblak <[email protected]> * fix: github clone shouldn't use URL directly from spec Signed-off-by: Olblak <[email protected]> * feat: add new githubsearch scm plugin * fix: move error message to debug * fix: github scm ut Signed-off-by: Olblak <[email protected]> * fix: github release UT Signed-off-by: Olblak <[email protected]> --------- Signed-off-by: Olblak <[email protected]>
1 parent c0a693c commit 60d57f4

File tree

13 files changed

+687
-80
lines changed

13 files changed

+687
-80
lines changed

‎pkg/core/engine/configuration.go‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,22 @@
11
package engine
22

33
import (
4+
"context"
45
"errors"
56
"fmt"
7+
"maps"
68
"os"
79
"strings"
810

911
"github.com/sirupsen/logrus"
1012
"github.com/updatecli/updatecli/pkg/core/cmdoptions"
1113
"github.com/updatecli/updatecli/pkg/core/config"
1214
"github.com/updatecli/updatecli/pkg/core/pipeline"
15+
"github.com/updatecli/updatecli/pkg/core/pipeline/scm"
1316
"github.com/updatecli/updatecli/pkg/core/reports"
1417
"github.com/updatecli/updatecli/pkg/core/result"
18+
"github.com/updatecli/updatecli/pkg/plugins/scms/github"
19+
"github.com/updatecli/updatecli/pkg/plugins/scms/githubsearch"
1520
)
1621

1722
// ReadConfigurations read every strategies configuration.
@@ -104,6 +109,62 @@ func (e *Engine) LoadConfigurations() error {
104109
continue
105110
}
106111

112+
// Load special scm configuration such as githubsearch that can generate multiple scm configurations
113+
// the generated scm configured must be ready before Updatecli start doing any operation such as
114+
// clone git repositories, using the autoddiscovery to detect potienial updates.
115+
// for id, loadedConfiguration := range loadedConfigurations {
116+
for i := 0; i < len(loadedConfigurations); i++ {
117+
118+
loadedConfiguration := loadedConfigurations[i]
119+
120+
for scmID, scmConfig := range loadedConfigurations[i].Spec.SCMs {
121+
switch scmConfig.Kind {
122+
case githubsearch.Kind:
123+
124+
logrus.Debugf("Processing githubsearch scm %q for potential multiple repository discovery", scmID)
125+
126+
ctx := context.Background()
127+
autodiscoveryScms, err := githubsearch.New(scmConfig.Spec)
128+
if err != nil {
129+
return fmt.Errorf("unable to instantiate githubsearch scm %q: %w", scmID, err)
130+
}
131+
discoveredSCms, err := autodiscoveryScms.ScmsGenerator(ctx)
132+
if err != nil {
133+
return fmt.Errorf("unable to generate scm specs for githubsearch scm %q: %w", scmID, err)
134+
}
135+
136+
if len(discoveredSCms) == 0 {
137+
// We need to trigger an error if the github search didn't discovered SCMs
138+
// Otherwise Updatecli will not know how to handle this kind of scm later.
139+
return fmt.Errorf("no scm discovered for githubsearch scm %q", scmID)
140+
}
141+
142+
scmConfig.Kind = github.Kind
143+
scmConfig.Spec = discoveredSCms[0]
144+
145+
loadedConfigurations[i].Spec.SCMs[scmID] = scmConfig
146+
147+
for _, spec := range discoveredSCms[1:] {
148+
newPipeline := loadedConfiguration
149+
150+
newPipeline.Spec.SCMs = make(map[string]scm.Config, len(loadedConfiguration.Spec.SCMs))
151+
maps.Copy(newPipeline.Spec.SCMs, loadedConfiguration.Spec.SCMs)
152+
153+
newSCM := newPipeline.Spec.SCMs[scmID]
154+
newSCM.Kind = github.Kind
155+
newSCM.Spec = spec
156+
157+
newPipeline.Spec.SCMs[scmID] = newSCM
158+
159+
loadedConfigurations = append(loadedConfigurations, newPipeline)
160+
logrus.Debugf("githubsearch scm %q added new pipeline configuration for repository %s/%s", scmID, spec.Owner, spec.Repository)
161+
}
162+
}
163+
}
164+
}
165+
166+
logrus.Debugf("Loaded %d pipeline configuration(s) from %q", len(loadedConfigurations), manifestFile)
167+
107168
for id := range loadedConfigurations {
108169
newPipeline := pipeline.Pipeline{}
109170
loadedConfiguration := loadedConfigurations[id]

‎pkg/core/pipeline/scm/config.go‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313
"github.com/updatecli/updatecli/pkg/plugins/scms/git"
1414
"github.com/updatecli/updatecli/pkg/plugins/scms/gitea"
1515
"github.com/updatecli/updatecli/pkg/plugins/scms/github"
16+
"github.com/updatecli/updatecli/pkg/plugins/scms/githubsearch"
1617
"github.com/updatecli/updatecli/pkg/plugins/scms/gitlab"
1718
"github.com/updatecli/updatecli/pkg/plugins/scms/stash"
1819
"github.com/updatecli/updatecli/pkg/plugins/utils/gitgeneric"
@@ -154,12 +155,13 @@ func (Config) JSONSchema() *jschema.Schema {
154155
type configAlias Config
155156

156157
anyOfSpec := map[string]interface{}{
157-
"bitbucket": &bitbucket.Spec{},
158-
"git": &git.Spec{},
159-
"gitea": &gitea.Spec{},
160-
"github": &github.Spec{},
161-
"gitlab": &gitlab.Spec{},
162-
"stash": &stash.Spec{},
158+
"bitbucket": &bitbucket.Spec{},
159+
"git": &git.Spec{},
160+
"gitea": &gitea.Spec{},
161+
"github": &github.Spec{},
162+
"gitlab": &gitlab.Spec{},
163+
"stash": &stash.Spec{},
164+
"githubsearch": &githubsearch.Spec{},
163165
}
164166

165167
return jsonschema.AppendOneOfToJsonSchema(configAlias{}, anyOfSpec)

‎pkg/core/pipeline/scm/main.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,8 @@ func (s *Scm) GenerateSCM() error {
122122
}
123123

124124
s.Handler = g
125+
case "githubsearch":
126+
// githubsearch scm kind is handled during engine preparation step
125127
default:
126128
return fmt.Errorf("scm of kind %q is not supported", s.Config.Kind)
127129
}

‎pkg/plugins/resources/githubrelease/main_test.go‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ func TestNew(t *testing.T) {
3434
Owner: "updatecli",
3535
Username: "joe",
3636
Token: "superSecretTOkenOfJoe",
37-
URL: "https://github.com",
37+
URL: "github.com",
3838
},
3939
},
4040
versionFilter: version.Filter{
@@ -60,7 +60,7 @@ func TestNew(t *testing.T) {
6060
Directory: path.Join(tmp.Directory, "updatecli", "updatecli"),
6161
Username: "joe",
6262
Token: "superSecretTOkenOfJoe",
63-
URL: "https://github.com",
63+
URL: "github.com",
6464
},
6565
},
6666
versionFilter: version.Filter{
@@ -86,7 +86,6 @@ func TestNew(t *testing.T) {
8686
Directory: "/home/updatecli",
8787
Username: "joe",
8888
Token: "superSecretTOkenOfJoe",
89-
URL: "https://github.com",
9089
},
9190
},
9291
versionFilter: version.Filter{
@@ -102,7 +101,6 @@ func TestNew(t *testing.T) {
102101
Owner: "updatecli",
103102
Username: "joe",
104103
Token: "superSecretTOkenOfJoe",
105-
URL: "github.com",
106104
Key: "commit",
107105
},
108106
wantErr: true,

‎pkg/plugins/scms/github/branch.go‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
package github
2+
3+
import (
4+
"context"
5+
"errors"
6+
"fmt"
7+
"strings"
8+
9+
"github.com/shurcooL/githubv4"
10+
"github.com/sirupsen/logrus"
11+
"github.com/updatecli/updatecli/pkg/plugins/scms/github/client"
12+
)
13+
14+
// branchesQuery defines the structure for the GraphQL query to list branches
15+
type branchesQuery struct {
16+
RateLimit RateLimit
17+
Repository struct {
18+
Refs struct {
19+
Nodes []struct {
20+
Name string
21+
}
22+
PageInfo struct {
23+
HasNextPage bool
24+
EndCursor githubv4.String
25+
}
26+
} `graphql:"refs(refPrefix: \"refs/heads/\", first: $first, after: $after)"`
27+
} `graphql:"repository(owner: $owner, name: $name)"`
28+
}
29+
30+
// ListBranches lists all branches from a specific repository using pagination
31+
func ListBranches(c client.Client, owner, repo string, retry int, ctx context.Context) ([]string, error) {
32+
var branches []string
33+
var after *githubv4.String
34+
35+
for {
36+
var q branchesQuery
37+
vars := map[string]interface{}{
38+
"owner": githubv4.String(owner),
39+
"name": githubv4.String(repo),
40+
"first": githubv4.Int(100),
41+
"after": after,
42+
}
43+
44+
err := c.Query(ctx, &q, vars)
45+
if err != nil {
46+
if strings.Contains(err.Error(), ErrAPIRateLimitExceeded) && retry < client.MaxRetry {
47+
rateLimit, err := queryRateLimit(c, ctx)
48+
if err != nil {
49+
logrus.Errorf("Error querying GitHub API rate limit: %s", err)
50+
}
51+
logrus.Debugln(rateLimit)
52+
if retry < client.MaxRetry {
53+
logrus.Warningf("GitHub API rate limit exceeded. Retrying... (%d/%d)", retry+1, client.MaxRetry)
54+
rateLimit.Pause()
55+
return ListBranches(c, owner, repo, retry+1, ctx)
56+
}
57+
return nil, errors.New(ErrAPIRateLimitExceededFinalAttempt)
58+
}
59+
return nil, fmt.Errorf("failed to list branches: %w", err)
60+
}
61+
62+
for _, node := range q.Repository.Refs.Nodes {
63+
branches = append(branches, node.Name)
64+
}
65+
66+
if q.Repository.Refs.PageInfo.HasNextPage {
67+
after = &q.Repository.Refs.PageInfo.EndCursor
68+
} else {
69+
break
70+
}
71+
}
72+
73+
return branches, nil
74+
}

‎pkg/plugins/scms/github/client/main.go‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,18 @@ package client
22

33
import (
44
"context"
5+
"errors"
6+
"fmt"
7+
"net/http"
8+
"net/url"
9+
"strings"
510

611
"github.com/shurcooL/githubv4"
12+
"github.com/sirupsen/logrus"
13+
"github.com/updatecli/updatecli/pkg/core/httpclient"
14+
"github.com/updatecli/updatecli/pkg/plugins/scms/github/app"
15+
"github.com/updatecli/updatecli/pkg/plugins/scms/github/token"
16+
"golang.org/x/oauth2"
717
)
818

919
// Client must be implemented by any GitHub query client (v4 API)
@@ -12,6 +22,87 @@ type Client interface {
1222
Mutate(ctx context.Context, m interface{}, input githubv4.Input, variables map[string]interface{}) error
1323
}
1424

25+
// ClientConfig defines the configuration required to create a GitHub client instance.
26+
type ClientConfig struct {
27+
Username string
28+
Client Client
29+
TokenSource oauth2.TokenSource
30+
URL string
31+
}
32+
1533
var (
34+
// MaxRetry defines the maximum number of retries when rate limit is exceeded
1635
MaxRetry = 3
1736
)
37+
38+
// New creates a new GitHub client instance based on the provided configuration.
39+
func New(configUsername, configToken string, configApp *app.Spec, configURL string) (client *ClientConfig, err error) {
40+
41+
URL := "github.com"
42+
if configURL != "" {
43+
URL = strings.TrimSpace(configURL)
44+
}
45+
46+
if !strings.HasPrefix(URL, "https://") && !strings.HasPrefix(URL, "http://") {
47+
URL = "https://" + URL
48+
}
49+
50+
// We first try to get a token source from the environment variable
51+
username, tokenSource, err := token.GetTokenSourceFromEnv()
52+
if err != nil {
53+
logrus.Debugf("no GitHub token found in environment variables: %s", err)
54+
}
55+
56+
// If no token source could be found in the environment variable
57+
// we try to get it from the configuration
58+
if tokenSource == nil {
59+
username, tokenSource, err = token.GetTokenSourceFromConfig(configUsername, configToken, configApp)
60+
if err != nil {
61+
return nil, fmt.Errorf("retrieving token source from configuration: %w", err)
62+
}
63+
}
64+
65+
if tokenSource == nil {
66+
username, tokenSource = token.GetFallbackTokenSourceFromEnv()
67+
}
68+
69+
// If the tokenSource is still nil at this point
70+
// it means that no valid token source could be found.
71+
// We log a debug message and return an error.
72+
if tokenSource == nil {
73+
logrus.Debugf(`GitHub token is not set, please refer to the documentation for more information:
74+
-> https://www.updatecli.io/docs/plugins/scm/github/
75+
`)
76+
return nil, errors.New("github token is not set")
77+
}
78+
79+
tokenSource = oauth2.ReuseTokenSource(nil, tokenSource)
80+
81+
clientContext := context.WithValue(
82+
context.Background(),
83+
oauth2.HTTPClient,
84+
httpclient.NewRetryClient().(*http.Client))
85+
86+
httpClient := oauth2.NewClient(clientContext, tokenSource)
87+
88+
var newClient Client
89+
90+
if strings.HasSuffix(URL, "github.com") {
91+
newClient = githubv4.NewClient(httpClient)
92+
} else {
93+
// For GH enterprise the GraphQL API path is /api/graphql
94+
// Cf https://docs.github.com/en/enterprise-cloud@latest/graphql/guides/managing-enterprise-accounts#3-setting-up-insomnia-to-use-the-github-graphql-api-with-enterprise-accounts
95+
graphqlURL, err := url.JoinPath(URL, "/api/graphql")
96+
if err != nil {
97+
return nil, fmt.Errorf("parsing GitHub Enterprise GraphQL URL: %w", err)
98+
}
99+
newClient = githubv4.NewEnterpriseClient(graphqlURL, httpClient)
100+
}
101+
102+
return &ClientConfig{
103+
Username: username,
104+
Client: newClient,
105+
TokenSource: tokenSource,
106+
URL: URL,
107+
}, nil
108+
}

0 commit comments

Comments
 (0)