package main import ( "database/sql" "encoding/xml" "fmt" _ "github.com/mattn/go-sqlite3" "io/ioutil" "net/http" "os" "strings" ) // Make up some data structures into which we can put our feed. // Bookmark defines the fundamental structure of the items to be archived. type Bookmark struct { // Required Title string `xml:"title"` Link string `xml:"link"` GUID string `xml:"guid"` // Optional PubDate string `xml:"pubDate"` Comments string `xml:"comments"` } // Feed defines the structure of the RSS feed exported from Pocket type Feed struct { XMLName xml.Name `xml:"rss"` Version string `xml:"version,attr"` // Required Title string `xml:"channel>title"` Link string `xml:"channel>link"` Description string `xml:"channel>description"` // Optional PubDate string `xml:"channel>pubDate"` BookmarkList []Bookmark `xml:"channel>item"` } // getDB opens a DB object and returns a usable DB instance func getDB(path string) (*sql.DB, error) { var fillDB bool _, err := os.Stat(path) if err != nil { fmt.Println("Database does not exist, creating and applying schema") fillDB = true } else { fillDB = false } db, err := sql.Open("sqlite3", path) if err != nil { fmt.Println("Could not open database") panic(err) } if fillDB { file, err := ioutil.ReadFile("./schema.sql") if err != nil { fmt.Println("database empty, but cold not read schema file") panic(err) } requests := strings.Split(string(file), ";") for _, request := range requests { _, err := db.Exec(request) if err != nil { fmt.Println("Could not execute:", request) panic(err) } } } return db, nil } func ingestJobExists(url string, db *sql.DB) bool { var count int err := db.QueryRow("SELECT count() FROM ingest where URL=?", url).Scan(&count) if err != nil { fmt.Println("Could not check ingest table for URL") panic(err) } if count > 0 { return true } return false } func bookmarkExists(url string, db *sql.DB) bool { var count int err := db.QueryRow("SELECT count() FROM bookmarks where URL=?", url).Scan(&count) if err != nil { fmt.Println("Could not check database for url") panic(err) } if count > 0 { return true } return false } func ingestURL(url string, db *sql.DB) sql.Result { if ingestJobExists(url, db) { fmt.Println("URL exists in ingest queue") row, err := db.Exec("SELECT * FROM ingest WHERE URL=?", url) if err != nil { fmt.Println("Could not get job from ingest queue") panic(err) } return row } row, err := db.Exec("INSERT INTO ingest(url) VALUES (?)", url) if err != nil { fmt.Println("Could not execute insert query") panic(err) } return row } func main() { fmt.Println("Launching Pocket Archive...") fmt.Println("Getting archive data from Pocket...") // Pull data from RSS feed. archiveURL := "https://getpocket.com/users/amdavidson/feed/read" resp, err := http.Get(archiveURL) if err != nil { fmt.Println("Could not get archived urls") panic(err) } defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) // Parse the feed f := Feed{} err = xml.Unmarshal(body, &f) if err != nil { fmt.Println("Could not parse feed") panic(err) } db, err := getDB("./bookmarks.db") if err != nil { fmt.Println("Could not open or create db") panic(err) } defer db.Close() for _, bookmark := range f.BookmarkList { if bookmarkExists(bookmark.GUID, db) == false { fmt.Printf("New bookmark url %s\n", bookmark.GUID) ingestURL(bookmark.GUID, db) } else { fmt.Printf("Already know about %s\n", bookmark.GUID) } } fmt.Println("Pocket Archive exiting.") }