fix() Cleanup strategy && Organisation 404 handler

Author: jsam

Makefile

@@ -1,4 +1,4 @@
-VERSION=0.1.3
+VERSION=0.1.4
 BINARY_NAME=ghbackup
 
 build:

config.yml

@@ -1,6 +1,6 @@
 keep_last_backup_days: 7
 organisations:
-  - kawago
+  - cawemo
   - flowing
   - bpmn-io
   - camunda

main.go

@@ -117,19 +117,42 @@ func (app *GithubBackup) uploadFileToS3(filePath string) {
 	}
 }
 
+func (app *GithubBackup) getS3Page(marker string) (*s3.ListObjectsOutput, error)  {
+	params := &s3.ListObjectsInput{
+		Bucket: aws.String(app.config.S3Bucket),
+	}
+
+	if len(marker) > 0 {
+		params.Marker = aws.String(marker)
+	}
+
+	return app.s3svc.ListObjects(params)
+}
+
 // cleanup method will delete old backups. Backup which are older then specified in config will be deleted.
 func (app *GithubBackup) cleanup() {
 	fmt.Println("[+] Starting CLEANUP.")
 
 	os.RemoveAll(strings.Split(TMP_REPO_PATH, "/")[0])
 	os.RemoveAll(app.createdAt)
 
-	params := &s3.ListObjectsInput{
-		Bucket: aws.String(app.config.S3Bucket),
+	var objRefs []*s3.Object
+	var marker string
+
+	for {
+		resp, _ := app.getS3Page(marker)
+		lastKey := resp.Contents[len(resp.Contents) - 1].Key
+		objRefs = append(objRefs, resp.Contents...)
+
+		if *resp.IsTruncated == true {
+			marker = *lastKey
+		} else {
+			break
+		}
 	}
-	resp, _ := app.s3svc.ListObjects(params)
 
-	for _, key := range resp.Contents {
+	fmt.Printf("[+] Found %d objects for cleanup.\n", len(objRefs))
+	for _, key := range objRefs {
 		fmt.Println(*key.Key)
 		ts, _ := ParseTime(strings.Split(*key.Key, "/")[0])
 		if int(time.Since(ts).Hours())  > app.config.KeepLastBackupDays * 24 {
@@ -197,7 +220,7 @@ func (app *GithubBackup) cloneRepository(repo *github.Repository, repoPath strin
 // downloadAll will fetch all repository endpoints for a given organisation and clone them to filesystem.
 func (app *GithubBackup) downloadAll(organisation string) {
 	repos, err := app.getRepositories(organisation)
-	checkErr(err)
+	if err != nil { return }
 
 	for _, repo := range repos {
 		path := fmt.Sprintf(TMP_REPO_PATH, app.createdAt, organisation, *repo.Name)