117 lines
2.6 KiB
Go
117 lines
2.6 KiB
Go
package ingest
|
|
|
|
import (
|
|
"fmt"
|
|
"github.com/mauidude/go-readability"
|
|
"io/ioutil"
|
|
"net/http"
|
|
)
|
|
|
|
func ingestJobExists(url string, db *sql.DB) bool {
|
|
var count int
|
|
|
|
err := db.QueryRow("SELECT count() FROM ingest where URL=?", url).Scan(&count)
|
|
if err != nil {
|
|
fmt.Println("Could not check ingest table for URL")
|
|
panic(err)
|
|
}
|
|
|
|
if count > 0 {
|
|
return true
|
|
}
|
|
|
|
return false
|
|
}
|
|
|
|
// createIngestJob puts the url into a table to queue for ingesting into the bookmark table.
|
|
func createIngestJob(url string, db *sql.DB) sql.Result {
|
|
if ingestJobExists(url, db) {
|
|
fmt.Println("URL exists in ingest queue")
|
|
row, err := db.Exec("SELECT * FROM ingest WHERE URL=?", url)
|
|
if err != nil {
|
|
fmt.Println("Could not get job from ingest queue")
|
|
panic(err)
|
|
}
|
|
return row
|
|
}
|
|
|
|
row, err := db.Exec("INSERT INTO ingest(url) VALUES (?)", url)
|
|
if err != nil {
|
|
fmt.Println("Could not execute insert query")
|
|
panic(err)
|
|
}
|
|
|
|
return row
|
|
}
|
|
|
|
// Ingests a URL into the bookmarks table
|
|
func ingestURL(url string, db *sql.DB) {
|
|
fmt.Println("Ingesting:", url)
|
|
resp, err := http.Get(url)
|
|
if err != nil {
|
|
fmt.Println("Could not access bookmarked url:", url)
|
|
panic(err)
|
|
}
|
|
defer resp.Body.Close()
|
|
html, err := ioutil.ReadAll(resp.Body)
|
|
|
|
doc, err := readability.NewDocument(string(html))
|
|
if err != nil {
|
|
fmt.Println("Could not parse site data for:", url)
|
|
}
|
|
fmt.Println("Content:", doc.Content())
|
|
}
|
|
func ingestJobExists(url string, db *sql.DB) bool {
|
|
var count int
|
|
|
|
err := db.QueryRow("SELECT count() FROM ingest where URL=?", url).Scan(&count)
|
|
if err != nil {
|
|
fmt.Println("Could not check ingest table for URL")
|
|
panic(err)
|
|
}
|
|
|
|
if count > 0 {
|
|
return true
|
|
}
|
|
|
|
return false
|
|
}
|
|
|
|
// createIngestJob puts the url into a table to queue for ingesting into the bookmark table.
|
|
func createIngestJob(url string, db *sql.DB) sql.Result {
|
|
if ingestJobExists(url, db) {
|
|
fmt.Println("URL exists in ingest queue")
|
|
row, err := db.Exec("SELECT * FROM ingest WHERE URL=?", url)
|
|
if err != nil {
|
|
fmt.Println("Could not get job from ingest queue")
|
|
panic(err)
|
|
}
|
|
return row
|
|
}
|
|
|
|
row, err := db.Exec("INSERT INTO ingest(url) VALUES (?)", url)
|
|
if err != nil {
|
|
fmt.Println("Could not execute insert query")
|
|
panic(err)
|
|
}
|
|
|
|
return row
|
|
}
|
|
|
|
// Ingests a URL into the bookmarks table
|
|
func ingestURL(url string, db *sql.DB) {
|
|
fmt.Println("Ingesting:", url)
|
|
resp, err := http.Get(url)
|
|
if err != nil {
|
|
fmt.Println("Could not access bookmarked url:", url)
|
|
panic(err)
|
|
}
|
|
defer resp.Body.Close()
|
|
html, err := ioutil.ReadAll(resp.Body)
|
|
|
|
doc, err := readability.NewDocument(string(html))
|
|
if err != nil {
|
|
fmt.Println("Could not parse site data for:", url)
|
|
}
|
|
fmt.Println("Content:", doc.Content())
|
|
}
|