Skip to content

Commit 90e47f6

Browse files
MeurillonGuillaumeMeurillonGuillaume
authored
Add support for creating repository indexes in JSON format (#739)
* repo: add support for JSON marshalling Signed-off-by: MeurillonGuillaume <MeurillonGuillaume@users.noreply.github.com> * cfg: add configuration for json index Signed-off-by: MeurillonGuillaume <MeurillonGuillaume@users.noreply.github.com> * repo: add tests for json and yaml index options Signed-off-by: MeurillonGuillaume <MeurillonGuillaume@users.noreply.github.com> --------- Signed-off-by: MeurillonGuillaume <MeurillonGuillaume@users.noreply.github.com> Co-authored-by: MeurillonGuillaume <MeurillonGuillaume@users.noreply.github.com>
1 parent f00683f commit 90e47f6

7 files changed

Lines changed: 76 additions & 23 deletions

File tree

‎cmd/chartmuseum/main.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,7 @@ func cliHandler(c *cli.Context) {
119119
WebTemplatePath: conf.GetString("web-template-path"),
120120
ArtifactHubRepoID: conf.GetStringMapString("artifact-hub-repo-id"),
121121
AlwaysRegenerateIndex: conf.GetBool("always-regenerate-chart-index"),
122+
JSONIndex: conf.GetBool("json-index"),
122123
}
123124

124125
server, err := newServer(options)

‎pkg/chartmuseum/server.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@ type (
8484
// AlwaysRegenerateIndex represents if the museum always return the up-to-date chart
8585
// which means that the GetChart will increase its latency , be careful to enable this .
8686
AlwaysRegenerateIndex bool
87+
JSONIndex bool
8788
}
8889

8990
// Server is a generic interface for web servers
@@ -151,6 +152,7 @@ func NewServer(options ServerOptions) (Server, error) {
151152
// EnforceSemver2 - see https://github.com/helm/chartmuseum/issues/485 for more info
152153
EnforceSemver2: options.EnforceSemver2,
153154
AlwaysRegenerateIndex: options.AlwaysRegenerateIndex,
155+
JSONIndex: options.JSONIndex,
154156
})
155157

156158
return server, err

‎pkg/chartmuseum/server/multitenant/cache.go‎

Lines changed: 21 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -152,11 +152,12 @@ func (server *MultiTenantServer) regenerateRepositoryIndexWorker(log cm_logger.L
152152
"repo", repo,
153153
)
154154
index := &cm_repo.Index{
155-
IndexFile: entry.RepoIndex.IndexFile,
156-
RepoName: repo,
157-
Raw: entry.RepoIndex.Raw,
158-
ChartURL: entry.RepoIndex.ChartURL,
159-
IndexLock: sync.RWMutex{},
155+
IndexFile: entry.RepoIndex.IndexFile,
156+
RepoName: repo,
157+
Raw: entry.RepoIndex.Raw,
158+
ChartURL: entry.RepoIndex.ChartURL,
159+
IndexLock: sync.RWMutex{},
160+
OutputJSON: server.JSONIndex,
160161
}
161162

162163
for _, object := range diff.Removed {
@@ -442,35 +443,41 @@ func (server *MultiTenantServer) newRepositoryIndex(log cm_logger.LoggingFn, rep
442443
}
443444

444445
if !server.UseStatefiles {
445-
return cm_repo.NewIndex(chartURL, repo, serverInfo)
446+
return cm_repo.NewIndex(chartURL, repo, serverInfo, server.JSONIndex)
446447
}
447448

448449
objectPath := pathutil.Join(repo, cm_repo.StatefileFilename)
449450
object, err := server.StorageBackend.GetObject(objectPath)
450451
if err != nil {
451-
return cm_repo.NewIndex(chartURL, repo, serverInfo)
452+
return cm_repo.NewIndex(chartURL, repo, serverInfo, server.JSONIndex)
452453
}
453454

454455
indexFile := &cm_repo.IndexFile{}
455-
err = yaml.Unmarshal(object.Content, indexFile)
456+
if json.Valid(object.Content) {
457+
err = json.Unmarshal(object.Content, indexFile)
458+
} else {
459+
err = yaml.Unmarshal(object.Content, indexFile)
460+
}
461+
456462
if err != nil {
457463
log(cm_logger.WarnLevel, "index-cache.yaml found but could not be parsed",
458464
"repo", repo,
459465
"error", err.Error(),
460466
)
461-
return cm_repo.NewIndex(chartURL, repo, serverInfo)
467+
return cm_repo.NewIndex(chartURL, repo, serverInfo, server.JSONIndex)
462468
}
463469

464470
log(cm_logger.DebugLevel, "index-cache.yaml loaded",
465471
"repo", repo,
466472
)
467473

468474
return &cm_repo.Index{
469-
IndexFile: indexFile,
470-
RepoName: repo,
471-
Raw: object.Content,
472-
ChartURL: chartURL,
473-
IndexLock: sync.RWMutex{},
475+
IndexFile: indexFile,
476+
RepoName: repo,
477+
Raw: object.Content,
478+
ChartURL: chartURL,
479+
IndexLock: sync.RWMutex{},
480+
OutputJSON: server.JSONIndex,
474481
}
475482
}
476483

‎pkg/chartmuseum/server/multitenant/server.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@ type (
7373
EnforceSemver2 bool
7474
WebTemplatePath string
7575
AlwaysRegenerateIndex bool
76+
JSONIndex bool
7677
}
7778

7879
ObjectsPerChartLimit struct {
@@ -106,6 +107,7 @@ type (
106107
// Deprecated: see https://github.com/helm/chartmuseum/issues/485 for more info
107108
EnforceSemver2 bool
108109
AlwaysRegenerateIndex bool
110+
JSONIndex bool
109111
}
110112

111113
tenantInternals struct {
@@ -166,6 +168,7 @@ func NewMultiTenantServer(options MultiTenantServerOptions) (*MultiTenantServer,
166168
WebTemplatePath: options.WebTemplatePath,
167169
ArtifactHubRepoID: options.ArtifactHubRepoID,
168170
AlwaysRegenerateIndex: options.AlwaysRegenerateIndex,
171+
JSONIndex: options.JSONIndex,
169172
}
170173

171174
if server.WebTemplatePath != "" {

‎pkg/config/vars.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,15 @@ var configVars = map[string]configVar{
138138
EnvVar: "DISABLE_STATEFILES",
139139
},
140140
},
141+
"json-index": {
142+
Type: boolType,
143+
Default: false,
144+
CLIFlag: cli.BoolFlag{
145+
Name: "json-index",
146+
Usage: "generates an index in JSON format, improves parsing performance for large index files",
147+
EnvVar: "JSON_INDEX",
148+
},
149+
},
141150
"allowoverwrite": {
142151
Type: boolType,
143152
Default: false,

‎pkg/repo/index.go‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ limitations under the License.
1717
package repo
1818

1919
import (
20+
"encoding/json"
2021
"sync"
2122
"time"
2223

@@ -51,27 +52,34 @@ type (
5152
Raw []byte `json:"c"`
5253
ChartURL string `json:"d"`
5354
IndexLock sync.RWMutex
55+
OutputJSON bool
5456
}
5557
)
5658

5759
// NewIndex creates a new instance of Index
58-
func NewIndex(chartURL string, repo string, serverInfo *ServerInfo) *Index {
60+
func NewIndex(chartURL string, repo string, serverInfo *ServerInfo, outputJSON bool) *Index {
5961
indexFile := &IndexFile{
6062
IndexFile: &helm_repo.IndexFile{},
6163
ServerInfo: serverInfo,
6264
}
63-
index := Index{indexFile, repo, []byte{}, chartURL, sync.RWMutex{}}
65+
index := Index{indexFile, repo, []byte{}, chartURL, sync.RWMutex{}, outputJSON}
6466
index.Entries = map[string]helm_repo.ChartVersions{}
6567
index.APIVersion = helm_repo.APIVersionV1
6668
index.Regenerate()
6769
return &index
6870
}
6971

7072
// Regenerate sorts entries in index file and sets current time for generated key
71-
func (index *Index) Regenerate() error {
73+
func (index *Index) Regenerate() (err error) {
7274
index.SortEntries()
7375
index.Generated = time.Now().Round(time.Second)
74-
raw, err := yaml.Marshal(index.IndexFile)
76+
77+
var raw []byte
78+
if index.OutputJSON {
79+
raw, err = json.Marshal(index.IndexFile)
80+
} else {
81+
raw, err = yaml.Marshal(index.IndexFile)
82+
}
7583
if err != nil {
7684
return err
7785
}

‎pkg/repo/index_test.go‎

Lines changed: 28 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,15 @@ limitations under the License.
1717
package repo
1818

1919
import (
20+
"encoding/json"
2021
"fmt"
2122
"testing"
2223
"time"
2324

2425
"strings"
2526

2627
"github.com/stretchr/testify/suite"
28+
"sigs.k8s.io/yaml"
2729

2830
"helm.sh/helm/v3/pkg/chart"
2931
helm_repo "helm.sh/helm/v3/pkg/repo"
@@ -51,7 +53,7 @@ func getChartVersion(name string, patch int, created time.Time) *helm_repo.Chart
5153
}
5254

5355
func (suite *IndexTestSuite) SetupSuite() {
54-
suite.Index = NewIndex("", "", &ServerInfo{})
56+
suite.Index = NewIndex("", "", &ServerInfo{}, false)
5557
now := time.Now()
5658
for _, name := range []string{"a", "b", "c"} {
5759
for i := 0; i < 10; i++ {
@@ -94,13 +96,13 @@ func (suite *IndexTestSuite) TestRemove() {
9496
}
9597

9698
func (suite *IndexTestSuite) TestChartURLs() {
97-
index := NewIndex("", "", &ServerInfo{})
99+
index := NewIndex("", "", &ServerInfo{}, false)
98100
chartVersion := getChartVersion("a", 0, time.Now())
99101
index.AddEntry(chartVersion)
100102
suite.Equal("charts/a-1.0.0.tgz",
101103
index.Entries["a"][0].URLs[0], "relative chart url")
102104

103-
index = NewIndex("http://mysite.com:8080", "", &ServerInfo{})
105+
index = NewIndex("http://mysite.com:8080", "", &ServerInfo{}, false)
104106
chartVersion = getChartVersion("a", 0, time.Now())
105107
index.AddEntry(chartVersion)
106108
suite.Equal("http://mysite.com:8080/charts/a-1.0.0.tgz",
@@ -109,16 +111,37 @@ func (suite *IndexTestSuite) TestChartURLs() {
109111

110112
func (suite *IndexTestSuite) TestServerInfo() {
111113
serverInfo := &ServerInfo{}
112-
index := NewIndex("", "", serverInfo)
114+
index := NewIndex("", "", serverInfo, false)
113115
suite.False(strings.Contains(string(index.Raw), "contextPath: /v1/helm"), "context path not in index")
114116

115117
serverInfo = &ServerInfo{
116118
ContextPath: "/v1/helm",
117119
}
118-
index = NewIndex("", "", serverInfo)
120+
index = NewIndex("", "", serverInfo, false)
119121
suite.True(strings.Contains(string(index.Raw), "contextPath: /v1/helm"), "context path is in index")
120122
}
121123

124+
func (suite *IndexTestSuite) TestYAMLIndex() {
125+
index := NewIndex("", "", &ServerInfo{}, false)
126+
chartVersion := getChartVersion("a", 0, time.Now())
127+
index.AddEntry(chartVersion)
128+
suite.NoError(index.Regenerate())
129+
130+
suite.False(json.Valid(index.Raw))
131+
suite.NoError(yaml.Unmarshal(index.Raw, &IndexFile{}))
132+
}
133+
134+
func (suite *IndexTestSuite) TestJSONIndex() {
135+
index := NewIndex("", "", &ServerInfo{}, true)
136+
chartVersion := getChartVersion("a", 0, time.Now())
137+
index.AddEntry(chartVersion)
138+
suite.NoError(index.Regenerate())
139+
140+
// Since YAML is a superset of JSON, any valid JSON should be valid YAML
141+
suite.True(json.Valid(index.Raw))
142+
suite.NoError(yaml.Unmarshal(index.Raw, &IndexFile{}))
143+
}
144+
122145
func TestIndexTestSuite(t *testing.T) {
123146
suite.Run(t, new(IndexTestSuite))
124147
}

0 commit comments

Comments
 (0)