Skip to content

Commit d5c79fc

Browse files
steve-r-weststeve-r-west
committed
WIP #458 
1 parent 7ab3685 commit d5c79fc

19 files changed

+410
-64
lines changed

‎cmd/create.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,7 @@ func NewCreateCommand(parentCmd *cobra.Command) func() {
125125
return err
126126
}
127127

128-
err = json.PrintJson(string(outputJson))
128+
err = json.PrintJsonToStdout(string(outputJson))
129129

130130
if err != nil {
131131
return err
@@ -146,7 +146,7 @@ func NewCreateCommand(parentCmd *cobra.Command) func() {
146146
}
147147
}
148148

149-
return json.PrintJson(body)
149+
return json.PrintJsonToStdout(body)
150150
}
151151

152152
}
@@ -343,7 +343,7 @@ func createInternal(ctx context.Context, overrides *httpclient.HttpParameterOver
343343

344344
// Check if error response
345345
if resp.StatusCode >= 400 && resp.StatusCode <= 600 {
346-
json.PrintJson(string(resBody))
346+
json.PrintJsonToStdout(string(resBody))
347347
return "", fmt.Errorf(resp.Status)
348348
}
349349

‎cmd/delete-all.go‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,19 @@ func deleteAllInternal(ctx context.Context, args []string) error {
103103
return err
104104
}
105105

106-
ids, totalCount, err := apihelper.GetResourceIdsFromHttpResponse(resp)
106+
if resp.StatusCode >= 400 {
107+
log.Warnf("Could not retrieve page of data, aborting")
108+
break
109+
}
110+
111+
bodyTxt, err := io.ReadAll(resp.Body)
112+
113+
if err != nil {
114+
return err
115+
}
116+
117+
ids, totalCount, err := apihelper.GetResourceIdsFromHttpResponse(bodyTxt)
118+
107119
resp.Body.Close()
108120

109121
allIds := make([][]id.IdableAttributes, 0)

‎cmd/delete.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@ func NewDeleteCommand(parentCmd *cobra.Command) func() {
9999
if err != nil {
100100
if body != "" {
101101
if !noBodyPrint {
102-
json.PrintJson(body)
102+
json.PrintJsonToStdout(body)
103103
}
104104
}
105105
return err
@@ -108,7 +108,7 @@ func NewDeleteCommand(parentCmd *cobra.Command) func() {
108108
if noBodyPrint {
109109
return nil
110110
} else {
111-
return json.PrintJson(body)
111+
return json.PrintJsonToStdout(body)
112112
}
113113
}
114114

‎cmd/get-all.go‎

Lines changed: 289 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,289 @@
1+
package cmd
2+
3+
import (
4+
"context"
5+
gojson "encoding/json"
6+
"fmt"
7+
"github.com/elasticpath/epcc-cli/external/apihelper"
8+
"github.com/elasticpath/epcc-cli/external/httpclient"
9+
"github.com/elasticpath/epcc-cli/external/id"
10+
"github.com/elasticpath/epcc-cli/external/json"
11+
"os"
12+
"sync"
13+
14+
"github.com/elasticpath/epcc-cli/external/resources"
15+
log "github.com/sirupsen/logrus"
16+
"github.com/spf13/cobra"
17+
"github.com/thediveo/enumflag"
18+
"io"
19+
"net/url"
20+
"reflect"
21+
"strconv"
22+
)
23+
24+
type OutputFormat enumflag.Flag
25+
26+
const (
27+
Jsonl OutputFormat = iota
28+
Csv
29+
EpccCli
30+
)
31+
32+
var OutputFormatIds = map[OutputFormat][]string{
33+
Jsonl: {"jsonl"},
34+
Csv: {"csv"},
35+
EpccCli: {"epcc-cli"},
36+
}
37+
38+
func NewGetAllCommand(parentCmd *cobra.Command) func() {
39+
40+
var getAll = &cobra.Command{
41+
Use: "get-all",
42+
Short: "Get all of a resource",
43+
SilenceUsage: false,
44+
RunE: func(cmd *cobra.Command, args []string) error {
45+
if len(args) == 0 {
46+
return fmt.Errorf("please specify a resource, epcc get-all [RESOURCE], see epcc delete-all --help")
47+
} else {
48+
return fmt.Errorf("invalid resource [%s] specified, see all with epcc delete-all --help", args[0])
49+
}
50+
},
51+
}
52+
53+
for _, resource := range resources.GetPluralResources() {
54+
if resource.GetCollectionInfo == nil {
55+
continue
56+
}
57+
58+
resourceName := resource.PluralName
59+
60+
var outputFile string
61+
var outputFormat OutputFormat
62+
63+
var getAllResourceCmd = &cobra.Command{
64+
Use: resourceName,
65+
Short: GetGetAllShort(resource),
66+
Hidden: false,
67+
RunE: func(cmd *cobra.Command, args []string) error {
68+
return getAllInternal(context.Background(), outputFormat, outputFile, append([]string{resourceName}, args...))
69+
},
70+
}
71+
72+
getAllResourceCmd.Flags().StringVarP(&outputFile, "output-file", "", "", "The file to output results to")
73+
74+
getAllResourceCmd.Flags().VarP(
75+
enumflag.New(&outputFormat, "output-format", OutputFormatIds, enumflag.EnumCaseInsensitive),
76+
"output-format", "",
77+
"sets output format; can be 'jsonl', 'csv', 'epcc-cli'")
78+
79+
getAll.AddCommand(getAllResourceCmd)
80+
}
81+
82+
parentCmd.AddCommand(getAll)
83+
return func() {}
84+
85+
}
86+
87+
func getAllInternal(ctx context.Context, outputFormat OutputFormat, outputFile string, args []string) error {
88+
// Find Resource
89+
resource, ok := resources.GetResourceByName(args[0])
90+
if !ok {
91+
return fmt.Errorf("could not find resource %s", args[0])
92+
}
93+
94+
if resource.GetCollectionInfo == nil {
95+
return fmt.Errorf("resource %s doesn't support GET collection", args[0])
96+
}
97+
98+
allParentEntityIds, err := getParentIds(ctx, resource)
99+
100+
if err != nil {
101+
return fmt.Errorf("could not retrieve parent ids for for resource %s, error: %w", resource.PluralName, err)
102+
}
103+
104+
if len(allParentEntityIds) == 1 {
105+
log.Debugf("Resource %s is a top level resource need to scan only one path to delete all resources", resource.PluralName)
106+
} else {
107+
log.Debugf("Resource %s is not a top level resource, need to scan %d paths to delete all resources", resource.PluralName, len(allParentEntityIds))
108+
}
109+
110+
var syncGroup = sync.WaitGroup{}
111+
112+
syncGroup.Add(1)
113+
114+
type idableAttributesWithType struct {
115+
id.IdableAttributes
116+
Type string `yaml:"type,omitempty" json:"type,omitempty"`
117+
EpccCliType string `yaml:"epcc_cli_type,omitempty" json:"epcc_cli_type,omitempty"`
118+
}
119+
120+
type msg struct {
121+
txt []byte
122+
id []idableAttributesWithType
123+
}
124+
var sendChannel = make(chan msg, 0)
125+
126+
var writer io.Writer
127+
if outputFile == "" {
128+
writer = os.Stdout
129+
} else {
130+
file, err := os.Create(outputFile)
131+
if err != nil {
132+
panic(err)
133+
}
134+
defer file.Close()
135+
writer = file
136+
}
137+
138+
outputWriter := func() {
139+
defer syncGroup.Done()
140+
141+
for msgs := 0; ; msgs++ {
142+
select {
143+
case result, ok := <-sendChannel:
144+
145+
if !ok {
146+
log.Debugf("Channel closed, we are done.")
147+
return
148+
}
149+
var obj interface{}
150+
err = gojson.Unmarshal(result.txt, &obj)
151+
152+
if err != nil {
153+
log.Errorf("Couldn't unmarshal JSON response %s due to error: %v", result, err)
154+
continue
155+
}
156+
157+
newObjs, err := json.RunJQWithArray(".data[]", obj)
158+
159+
if err != nil {
160+
log.Errorf("Couldn't process response %s due to error: %v", result, err)
161+
continue
162+
}
163+
164+
for _, newObj := range newObjs {
165+
166+
wrappedObj := map[string]interface{}{
167+
"data": newObj,
168+
"meta": map[string]interface{}{
169+
"_epcc_cli_parent_resources": result.id,
170+
},
171+
}
172+
173+
line, err := gojson.Marshal(&wrappedObj)
174+
175+
if err != nil {
176+
log.Errorf("Could not create JSON for %s, error: %v", line, err)
177+
continue
178+
}
179+
180+
_, err = writer.Write(line)
181+
182+
if err != nil {
183+
log.Errorf("Could not save line %s, error: %v", line, err)
184+
continue
185+
}
186+
187+
_, err = writer.Write([]byte{10})
188+
189+
if err != nil {
190+
log.Errorf("Could not save line %s, error: %v", line, err)
191+
continue
192+
}
193+
194+
}
195+
196+
}
197+
}
198+
}
199+
200+
go outputWriter()
201+
202+
for _, parentEntityIds := range allParentEntityIds {
203+
lastIds := make([][]id.IdableAttributes, 1)
204+
for offset := 0; offset <= 10000; offset += 100 {
205+
resourceURL, err := resources.GenerateUrlViaIdableAttributes(resource.GetCollectionInfo, parentEntityIds)
206+
207+
if err != nil {
208+
return err
209+
}
210+
211+
types, err := resources.GetSingularTypesOfVariablesNeeded(resource.GetCollectionInfo.Url)
212+
213+
if err != nil {
214+
return err
215+
}
216+
217+
params := url.Values{}
218+
params.Add("page[limit]", "100")
219+
params.Add("page[offset]", strconv.Itoa(offset))
220+
221+
resp, err := httpclient.DoRequest(ctx, "GET", resourceURL, params.Encode(), nil)
222+
223+
if err != nil {
224+
return err
225+
}
226+
227+
if resp.StatusCode >= 400 {
228+
log.Warnf("Could not retrieve page of data, aborting")
229+
230+
break
231+
}
232+
233+
bodyTxt, err := io.ReadAll(resp.Body)
234+
235+
if err != nil {
236+
237+
return err
238+
}
239+
240+
ids, totalCount, err := apihelper.GetResourceIdsFromHttpResponse(bodyTxt)
241+
resp.Body.Close()
242+
243+
allIds := make([][]id.IdableAttributes, 0)
244+
for _, id := range ids {
245+
allIds = append(allIds, append(parentEntityIds, id))
246+
}
247+
248+
if reflect.DeepEqual(allIds, lastIds) {
249+
log.Warnf("Data on the previous two pages did not change. Does this resource support pagination? Aborting export", resource.PluralName, len(allIds))
250+
251+
break
252+
} else {
253+
lastIds = allIds
254+
}
255+
256+
idsWithType := make([]idableAttributesWithType, len(types))
257+
258+
for i, t := range types {
259+
idsWithType[i].IdableAttributes = parentEntityIds[i]
260+
idsWithType[i].EpccCliType = t
261+
idsWithType[i].Type = resources.MustGetResourceByName(t).JsonApiType
262+
}
263+
264+
sendChannel <- msg{
265+
bodyTxt,
266+
idsWithType,
267+
}
268+
269+
if len(allIds) == 0 {
270+
log.Infof("Total ids retrieved for %s in %s is %d, we are done", resource.PluralName, resourceURL, len(allIds))
271+
272+
break
273+
} else {
274+
if totalCount >= 0 {
275+
log.Infof("Total number of %s in %s is %d", resource.PluralName, resourceURL, totalCount)
276+
} else {
277+
log.Infof("Total number %s in %s is unknown", resource.PluralName, resourceURL)
278+
}
279+
}
280+
281+
}
282+
}
283+
284+
close(sendChannel)
285+
286+
syncGroup.Wait()
287+
288+
return nil
289+
}

‎cmd/get.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -193,7 +193,7 @@ func NewGetCommand(parentCmd *cobra.Command) func() {
193193
return err
194194
}
195195

196-
err = json.PrintJson(string(outputJson))
196+
err = json.PrintJsonToStdout(string(outputJson))
197197

198198
if err != nil {
199199
return err
@@ -214,7 +214,7 @@ func NewGetCommand(parentCmd *cobra.Command) func() {
214214
}
215215
}
216216

217-
printError := json.PrintJson(body)
217+
printError := json.PrintJsonToStdout(body)
218218

219219
if retriesFailedError != nil {
220220
return retriesFailedError
@@ -326,7 +326,7 @@ func getInternal(ctx context.Context, overrides *httpclient.HttpParameterOverrid
326326

327327
// Check if error response
328328
if resp.StatusCode >= 400 && resp.StatusCode <= 600 {
329-
json.PrintJson(string(body))
329+
json.PrintJsonToStdout(string(body))
330330
return "", fmt.Errorf(resp.Status)
331331
}
332332

‎cmd/helper.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -243,6 +243,10 @@ func GetDeleteAllShort(resource resources.Resource) string {
243243
return fmt.Sprintf("Calls DELETE %s for every resource in GET %s", GetHelpResourceUrls(resource.DeleteEntityInfo.Url), GetHelpResourceUrls(resource.GetCollectionInfo.Url))
244244
}
245245

246+
func GetGetAllShort(resource resources.Resource) string {
247+
return fmt.Sprintf("Calls GET %s and iterates over all pages and parent resources (if applicable)", GetHelpResourceUrls(resource.GetCollectionInfo.Url))
248+
}
249+
246250
func GetGetLong(resourceName string, resourceUrl string, usageGetType string, completionVerb int, urlInfo *resources.CrudEntityInfo, resource resources.Resource) string {
247251

248252
if DisableLongOutput {

0 commit comments

Comments
 (0)