93 lines
1.9 KiB
Go
93 lines
1.9 KiB
Go
package main
|
|
|
|
import (
|
|
"database/sql"
|
|
"encoding/json"
|
|
"encoding/xml"
|
|
"fmt"
|
|
"github.com/mauidude/go-readability"
|
|
"io/ioutil"
|
|
"net/http"
|
|
)
|
|
|
|
func pullPocket(db *sql.DB) {
|
|
fmt.Println("Getting archive data from Pocket...")
|
|
|
|
// Pull data from RSS feed.
|
|
archiveURL := "https://getpocket.com/users/amdavidson/feed/read"
|
|
|
|
resp, err := http.Get(archiveURL)
|
|
if err != nil {
|
|
fmt.Println("Could not get archived urls")
|
|
panic(err)
|
|
}
|
|
|
|
defer resp.Body.Close()
|
|
body, err := ioutil.ReadAll(resp.Body)
|
|
|
|
// Parse the feed
|
|
f := Feed{}
|
|
err = xml.Unmarshal(body, &f)
|
|
if err != nil {
|
|
fmt.Println("Could not parse feed")
|
|
panic(err)
|
|
}
|
|
|
|
for _, bookmark := range f.BookmarkList {
|
|
if bookmarkExists(bookmark.URL, db) == false {
|
|
fmt.Printf("New bookmark url %s\n", bookmark.URL)
|
|
createIngestJob(bookmark.URL, db)
|
|
} else {
|
|
fmt.Printf("Already know about %s\n", bookmark.URL)
|
|
}
|
|
}
|
|
}
|
|
|
|
func getMercury(url string) (Mercury, error) {
|
|
|
|
var m Mercury
|
|
|
|
client := http.Client{}
|
|
|
|
reqURL := fmt.Sprintf("https://mercury.postlight.com/parser?url=%s", url)
|
|
|
|
req, err := http.NewRequest("GET", reqURL, nil)
|
|
if err != nil {
|
|
fmt.Println("Could not make new request")
|
|
return m, err
|
|
}
|
|
|
|
req.Header.Add("x-api-key", "BM0BDRUyRbZCruv0VPJr3N9DBGGhW1GixHxS1sEB")
|
|
|
|
mercuryRet, err := client.Do(req)
|
|
if err != nil {
|
|
fmt.Println("Could not request data from Mercury")
|
|
return m, err
|
|
}
|
|
|
|
mercuryJSON, _ := ioutil.ReadAll(mercuryRet.Body)
|
|
|
|
err = json.Unmarshal(mercuryJSON, &m)
|
|
if err != nil {
|
|
fmt.Println("Could not parse Mercury output")
|
|
return m, err
|
|
}
|
|
|
|
return m, nil
|
|
}
|
|
|
|
func getReadability(url string) string {
|
|
resp, err := http.Get(url)
|
|
if err != nil {
|
|
return "Could not get readability data. Could not reach url."
|
|
}
|
|
defer resp.Body.Close()
|
|
html, err := ioutil.ReadAll(resp.Body)
|
|
|
|
doc, err := readability.NewDocument(string(html))
|
|
if err != nil {
|
|
return "Could not get readability data. Could not parse retrieved data."
|
|
}
|
|
|
|
return doc.Content()
|
|
}
|