interim commit on github backups

.gitignore

@@ -0,0 +1 @@
+./data

cmd/github.go

@@ -5,17 +5,22 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
+	"os"
 	"strings"
+	"time"
+	"sync"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
+	"gopkg.in/src-d/go-git.v4"
+	bolt "go.etcd.io/bbolt"
 )
 
 // githubCmd represents the github command
 var githubCmd = &cobra.Command{
 	Use:   "github",
-	Short: "Backup your Github account",
+	Short: "Github archivist support",
-	Long:  `Backup your Github account to local storage`,
+	Long:  `Archivist tools to backup your Github account to local storage`,
 	Run: func(cmd *cobra.Command, args []string) {
 		err := ghList()
 		if err != nil {
@@ -29,6 +34,7 @@ var githubToken string
 
 func init() {
 	rootCmd.AddCommand(githubCmd)
+	githubCmd.AddCommand(backupCmd)
 
 	githubCmd.PersistentFlags().StringVar(&githubUser, "github-user", "", "The github user to backup")
 	viper.BindPFlag("github.user", githubCmd.PersistentFlags().Lookup("github-user"))
@@ -37,12 +43,22 @@ func init() {
 
 }
 
+var backupCmd = &cobra.Command{
+	Use:   "backup",
+	Short: "Run a backup on your Github account",
+	Long:  ``,
+	Run: func(cmd *cobra.Command, args []string) {
+		err := ghBackup()
+		if err != nil {
+			fmt.Println(err)
+		}
+	},
+}
+
 type repo struct {
 	FullName    string `json:"full_name"`
-	CloneHTTPS  string `json:"clone_url"`
+	UpdatedDate time.Time `json:"updated_at"`
-	UpdatedDate string `json:"updated_at"`
+	CreatedDate time.Time `json:"created_at"`
-	CreatedDate string `json:"created_at"`
-	ArchiveURL  string `json:"archive_url"`
 }
 
 func ghQuery(url string) ([]byte, string, error) {
@@ -115,10 +131,82 @@ func ghList() error {
 	for _, r := range repos {
 
 		fmt.Println(r.FullName)
-		fmt.Println("\tUpdated:", r.UpdatedDate)
+		if Verbose {
-		fmt.Println("\tCreated:", r.CreatedDate)
+			fmt.Println("\tUpdated:", r.UpdatedDate)
-
+			fmt.Println("\tCreated:", r.CreatedDate)
+		}
 	}
 
 	return nil
 }
+
+func ghBackup() error {
+	if Verbose {
+		fmt.Println("Backup up", C.Green("Github"), "repos for", C.Magenta(viper.Get("github.user")))
+	}
+
+	repos, err := ghGetRepos() 
+	if err != nil {
+		return err
+	}
+
+	var ghBackupDate time.Time
+
+	db.View(func (tx *bolt.Tx) error {
+		b := tx.Bucket([]byte("Github"))
+		ghBackupDate := b.Get([]byte("last-backup"))
+		return nil
+	})
+
+	tasks := make(chan repo)
+
+	// ToDo fix below statements completely non-functioning
+
+	for _, r := range repos {
+		if r.UpdatedDate > ghBackupDate {
+			tasks <- r
+		}
+	}
+
+	var wg sync.WaitGroup 
+	for i := 0; i < 5; i++ {
+		wg.Add(1)
+		go func () {
+			for repo := range tasks {
+				ghBackupRepo(repo)
+			}
+		}
+	}
+	wg.Wait()
+
+	// End ToDo
+}
+
+func ghBackupRepo(repo) error {
+	if Verbose {
+		fmt.Println("Backing up ", repo.FullName)
+	}
+
+	cloneURL := "https://" + viper.GetString("github.token") + "@github.com/" + repo.FullName
+
+	if Verbose {
+		_, err := git.PlainClone(DataDir+"github", false, &git.CloneOptions{
+			URL:      cloneURL,
+			Progress: os.Stdout,
+		})
+		if err != nil {
+			return err
+		}
+	} else {
+		_, err := git.PlainClone(DataDir+"github", false, &git.CloneOptions{
+			URL: cloneURL,
+		})
+		if err != nil {
+			return err
+		}
+	}
+
+	if Verbose {
+		fmt.Println("Done backing up", repo.FullName)
+	}
+}

cmd/root.go

@@ -8,12 +8,15 @@ import (
 	"github.com/logrusorgru/aurora"
 	homedir "github.com/mitchellh/go-homedir"
 	"github.com/spf13/viper"
+	bolt "go.etcd.io/bbolt"
 )
 
+var db bolt.DB
 var cfgFile string
 var Verbose bool
 var C aurora.Aurora
 var color bool
+var DataDir string
 
 // rootCmd represents the base command when called without any subcommands
 var rootCmd = &cobra.Command{
@@ -25,11 +28,6 @@ var rootCmd = &cobra.Command{
 // Execute adds all child commands to the root command and sets flags appropriately.
 // This is called by main.main(). It only needs to happen once to the rootCmd.
 func Execute() {
-	if color {
-		fmt.Println("Use Color")
-	} else {
-		fmt.Println("Do not use color")
-	}
 	C = aurora.NewAurora(color)
 	if err := rootCmd.Execute(); err != nil {
 		fmt.Println(err)
@@ -45,6 +43,14 @@ func init() {
 	viper.BindPFlag("verbose", rootCmd.PersistentFlags().Lookup("verbose"))
 	rootCmd.PersistentFlags().BoolVar(&color, "colors", true, "Don't be flashy")
 	viper.BindPFlag("colors", rootCmd.PersistentFlags().Lookup("colors"))
+	rootCmd.PersistentFlags().StringVarP(&DataDir, "data", "d", "./data", "Location to store data.")
+	viper.BindPFlag("data", rootCmd.PersistentFlags().Lookup("data"))
+
+	db, err := bolt.Open(data+"/archivist.db", 0600, nil)
+	if err != nil {
+		fmt.Println("Cannot open directory")
+	}
+	defer db.Close()
 }
 
 // initConfig reads in config file and ENV variables if set.