Compare commits
No commits in common. "684ee15a9557cd4cb405eb1a3729fc49d8f6fad8" and "df939a1b50fe942f230da2a7d7476ce3e16b3396" have entirely different histories.
684ee15a95
...
df939a1b50
134
feed/feed.go
134
feed/feed.go
|
|
@ -1,7 +1,6 @@
|
|||
package feed
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
|
|
@ -48,14 +47,6 @@ type FeedItem struct {
|
|||
RawText string
|
||||
}
|
||||
|
||||
type SitePage struct {
|
||||
Url string
|
||||
Title string
|
||||
Root *html.Node
|
||||
Errors []error
|
||||
ErrStr string
|
||||
}
|
||||
|
||||
func fetchPage(url string) (string, error) {
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
|
|
@ -94,7 +85,7 @@ func parseTime(timestr string) (time.Time, error) {
|
|||
return pagetime, err
|
||||
}
|
||||
}
|
||||
return pagetime, fmt.Errorf("%s is in an invalid format", timestr)
|
||||
return pagetime, fmt.Errorf("Error parsing time: invalid format")
|
||||
}
|
||||
|
||||
func getHtmlElement(doc *html.Node, tag string) (*html.Node, error) {
|
||||
|
|
@ -116,36 +107,29 @@ func getHtmlElement(doc *html.Node, tag string) (*html.Node, error) {
|
|||
return element, nil
|
||||
}
|
||||
|
||||
func getAllElements(doc *html.Node, tag string) ([]*html.Node, error) {
|
||||
var f func(*html.Node, string)
|
||||
elements := make([]*html.Node, 0)
|
||||
f = func(n *html.Node, s string) {
|
||||
if n.Type == html.ElementNode && n.Data == s{
|
||||
elements = append(elements, n)
|
||||
return
|
||||
}
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
f(c, tag)
|
||||
}
|
||||
}
|
||||
f(doc, tag)
|
||||
if len(elements) == 0 {
|
||||
return nil, fmt.Errorf("no <%s> element found", tag)
|
||||
}
|
||||
return elements, nil
|
||||
}
|
||||
|
||||
func NewFeedItem(url string, article *html.Node) (*FeedItem, error) {
|
||||
var articleBuilder strings.Builder
|
||||
html.Render(&articleBuilder, article)
|
||||
item := FeedItem{
|
||||
Url: url,
|
||||
RawText: articleBuilder.String(),
|
||||
}
|
||||
|
||||
etime, err := getHtmlElement(article, "time")
|
||||
func (f *FeedItem) ParseContent(content string) error {
|
||||
doc, err := html.Parse(strings.NewReader(content))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return fmt.Errorf("Error parsing HTML: %w", err)
|
||||
}
|
||||
earticle, err := getHtmlElement(doc, "article")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
etitle, err := getHtmlElement(doc, "title")
|
||||
if err != nil {
|
||||
f.Title = ""
|
||||
} else {
|
||||
f.Title = etitle.FirstChild.Data
|
||||
}
|
||||
|
||||
var articleBuilder strings.Builder
|
||||
html.Render(&articleBuilder, earticle)
|
||||
f.RawText = articleBuilder.String()
|
||||
|
||||
etime, err := getHtmlElement(earticle, "time")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var pubTime time.Time
|
||||
for _, d := range etime.Attr {
|
||||
|
|
@ -153,73 +137,26 @@ func NewFeedItem(url string, article *html.Node) (*FeedItem, error) {
|
|||
pubTime, err = parseTime(d.Val)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Error parsing time: %w", err)
|
||||
return fmt.Errorf("Error parsing time: %w", err)
|
||||
}
|
||||
item.PubTime = pubTime
|
||||
f.PubTime = pubTime
|
||||
}
|
||||
|
||||
eh1, _ := getHtmlElement(article, "h1")
|
||||
eh2, _ := getHtmlElement(article, "h2")
|
||||
eh3, _ := getHtmlElement(article, "h3")
|
||||
if eh1 != nil {
|
||||
item.Title = eh1.FirstChild.Data
|
||||
// TODO: handle <a>
|
||||
} else if eh2 != nil {
|
||||
item.Title = eh2.FirstChild.Data
|
||||
} else if eh3 != nil {
|
||||
item.Title = eh3.FirstChild.Data
|
||||
} else {
|
||||
item.Title = pubTime.Format("Jan 02 2006")
|
||||
}
|
||||
|
||||
return &item, nil
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *SitePage) Parse() ([]*FeedItem, error) {
|
||||
items := make([]*FeedItem, 0)
|
||||
articles, err := getAllElements(p.Root, "article")
|
||||
if err != nil {
|
||||
return nil, errors.New("No article elements found")
|
||||
}
|
||||
for _, article := range articles {
|
||||
item, parseErr := NewFeedItem(p.Url, article)
|
||||
if parseErr != nil {
|
||||
p.Errors = append(p.Errors, parseErr)
|
||||
} else {
|
||||
items = append(items, item)
|
||||
}
|
||||
}
|
||||
if len(p.Errors) > 0 {
|
||||
errorStrs := make([]string, 0)
|
||||
for _, perr := range p.Errors {
|
||||
errorStrs = append(errorStrs, perr.Error())
|
||||
}
|
||||
p.ErrStr = errors.New(strings.Join(errorStrs, "\n")).Error()
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
func NewSitePage(url string) (*SitePage, error) {
|
||||
func NewFeedItem(url string) (*FeedItem, error) {
|
||||
rawhtml, err := fetchPage(url)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Could not fetch page '%s': %w", url, err)
|
||||
}
|
||||
nodeRoot, err := html.Parse(strings.NewReader(rawhtml))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("Error parsing HTML: %w", err)
|
||||
}
|
||||
page := SitePage{
|
||||
item := FeedItem{
|
||||
Url: url,
|
||||
Root: nodeRoot,
|
||||
Errors: make([]error, 0),
|
||||
}
|
||||
nodeTitle, err := getHtmlElement(nodeRoot, "title")
|
||||
err = item.ParseContent(rawhtml);
|
||||
if err != nil {
|
||||
page.Title = url
|
||||
} else {
|
||||
page.Title = nodeTitle.FirstChild.Data
|
||||
return nil, fmt.Errorf("Could not parse feed item: %w", err)
|
||||
}
|
||||
return &page, nil
|
||||
return &item, nil
|
||||
}
|
||||
|
||||
func NewFeedInfo(name, base_url, desc, author string, page_urls...string) (*FeedInfo, error) {
|
||||
|
|
@ -231,16 +168,11 @@ func NewFeedInfo(name, base_url, desc, author string, page_urls...string) (*Feed
|
|||
Errors: make(map[string]string, 10),
|
||||
}
|
||||
for _,url := range info.PageUrls {
|
||||
page, err := NewSitePage(url)
|
||||
if err != nil {
|
||||
info.Errors[url] = err.Error()
|
||||
}
|
||||
pageItems, err := page.Parse()
|
||||
item, err := NewFeedItem(url)
|
||||
if err != nil {
|
||||
info.Errors[url] = err.Error()
|
||||
} else {
|
||||
info.Items = append(info.Items, pageItems...)
|
||||
info.Errors[url] = page.ErrStr
|
||||
info.Items = append(info.Items, item)
|
||||
}
|
||||
}
|
||||
return &info, nil
|
||||
|
|
|
|||
|
|
@ -1,12 +1,9 @@
|
|||
package feed_test
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.32bit.cafe/yequari/webweav.ing/feed"
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
func TestTimeParsing(t *testing.T) {
|
||||
|
|
@ -22,38 +19,35 @@ func TestTimeParsing(t *testing.T) {
|
|||
}{
|
||||
{
|
||||
"YYYY-MM-DD",
|
||||
`<article><time datetime="2004-05-14">May 14 2004</time>hello world</article>`,
|
||||
`<html><head></head><body><article><time datetime="2004-05-14">May 14 2004</time>hello world</article></body></html>`,
|
||||
time.DateOnly,
|
||||
},
|
||||
{
|
||||
"YYYY-MM-DD HH:MM",
|
||||
`<article><time datetime="2004-05-14 07:30">May 14 2004</time>hello world</article>`,
|
||||
`<html><head></head><body><article><time datetime="2004-05-14 07:30">May 14 2004</time>hello world</article></body></html>`,
|
||||
"2006-01-02 15:04",
|
||||
},
|
||||
{
|
||||
"YYYY-MM-DD HH:MM:SS",
|
||||
`<article><time datetime="2004-05-14 07:30:55">May 14 2004</time>hello world</article>`,
|
||||
`<html><head></head><body><article><time datetime="2004-05-14 07:30:55">May 14 2004</time>hello world</article></body></html>`,
|
||||
"2006-01-02 15:04:05",
|
||||
},
|
||||
{
|
||||
"YYYY-MM-DDTHH:MM:SS",
|
||||
`<article><time datetime="2004-05-14T07:30:55">May 14 2004</time>hello world</article>`,
|
||||
`<html><head></head><body><article><time datetime="2004-05-14T07:30:55">May 14 2004</time>hello world</article></body></html>`,
|
||||
"2006-01-02T15:04:05",
|
||||
},
|
||||
{
|
||||
"YYYY-MM-DDTHH:MM",
|
||||
`<article><time datetime="2004-05-14T07:30">May 14 2004</time>hello world</article>`,
|
||||
`<html><head></head><body><article><time datetime="2004-05-14T07:30">May 14 2004</time>hello world</article></body></html>`,
|
||||
"2006-01-02T15:04",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func (t *testing.T) {
|
||||
html, err := html.Parse(strings.NewReader(tt.input))
|
||||
if err != nil {
|
||||
t.Errorf("error: %s", err)
|
||||
}
|
||||
item, err := feed.NewFeedItem("", html)
|
||||
item := feed.FeedItem{}
|
||||
err := item.ParseContent(tt.input)
|
||||
if err != nil {
|
||||
t.Errorf("error: %s", err)
|
||||
}
|
||||
|
|
@ -67,106 +61,38 @@ func TestTimeParsing(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestArticleParsing(t *testing.T) {
|
||||
|
||||
func TestParseFeedItem(t *testing.T) {
|
||||
testDate, err := time.Parse("2006-Jan-02", "2004-May-14")
|
||||
if err != nil {
|
||||
t.Errorf("creating test date failed: %s", err)
|
||||
}
|
||||
testDate2, err := time.Parse("2006-Jan-02", "2004-May-07")
|
||||
if err != nil {
|
||||
t.Errorf("creating test date failed: %s", err)
|
||||
}
|
||||
var tests = []struct {
|
||||
name string
|
||||
input string
|
||||
want_time []*time.Time
|
||||
want_article []string
|
||||
want_title []string
|
||||
want_time *time.Time
|
||||
want_article string
|
||||
}{
|
||||
{
|
||||
"article and time stripped out of basic HTML",
|
||||
"<html><head></head><body><article><time datetime=\"2004-05-14\">May 14 2004</time>hello world</article></body></html>",
|
||||
[]*time.Time{&testDate},
|
||||
[]string{"<article><time datetime=\"2004-05-14\">May 14 2004</time>hello world</article>"},
|
||||
[]string{"May 14 2004"},
|
||||
},
|
||||
{
|
||||
"multiple articles",
|
||||
"<html><head></head><body><article><time datetime=\"2004-05-14\">May 14 2004</time>hello world</article><article><time datetime=\"2004-05-07\">May 7 2004</time>this is a second article</article></body></html>",
|
||||
[]*time.Time{&testDate, &testDate2},
|
||||
[]string{"<article><time datetime=\"2004-05-14\">May 14 2004</time>hello world</article>", "<article><time datetime=\"2004-05-07\">May 7 2004</time>this is a second article</article>"},
|
||||
[]string{"May 14 2004", "May 07 2004"},
|
||||
},
|
||||
{
|
||||
"article with h1",
|
||||
"<html><head></head><body><article><time datetime=\"2004-05-14\">May 14 2004</time><h1>Hello</h1>hello world</article></body></html>",
|
||||
[]*time.Time{&testDate},
|
||||
[]string{"<article><time datetime=\"2004-05-14\">May 14 2004</time><h1>Hello</h1>hello world</article>"},
|
||||
[]string{"Hello"},
|
||||
},
|
||||
{
|
||||
"article with h2",
|
||||
"<html><head></head><body><article><time datetime=\"2004-05-14\">May 14 2004</time><h2>Hello</h2>hello world</article></body></html>",
|
||||
[]*time.Time{&testDate},
|
||||
[]string{"<article><time datetime=\"2004-05-14\">May 14 2004</time><h2>Hello</h2>hello world</article>"},
|
||||
[]string{"Hello"},
|
||||
},
|
||||
{
|
||||
"article with h3",
|
||||
"<html><head></head><body><article><time datetime=\"2004-05-14\">May 14 2004</time><h3>Hello</h3>hello world</article></body></html>",
|
||||
[]*time.Time{&testDate},
|
||||
[]string{"<article><time datetime=\"2004-05-14\">May 14 2004</time><h3>Hello</h3>hello world</article>"},
|
||||
[]string{"Hello"},
|
||||
},
|
||||
{
|
||||
"article with h1 and h2",
|
||||
"<html><head></head><body><article><time datetime=\"2004-05-14\">May 14 2004</time><h1>Hello</h1><h2>World</h2>hello world</article></body></html>",
|
||||
[]*time.Time{&testDate},
|
||||
[]string{"<article><time datetime=\"2004-05-14\">May 14 2004</time><h1>Hello</h1><h2>World</h2>hello world</article>"},
|
||||
[]string{"Hello"},
|
||||
},
|
||||
{
|
||||
"article with h2 and h3",
|
||||
"<html><head></head><body><article><time datetime=\"2004-05-14\">May 14 2004</time><h3>Hello</h3><h2>World</h2>hello world</article></body></html>",
|
||||
[]*time.Time{&testDate},
|
||||
[]string{"<article><time datetime=\"2004-05-14\">May 14 2004</time><h3>Hello</h3><h2>World</h2>hello world</article>"},
|
||||
[]string{"World"},
|
||||
},
|
||||
{
|
||||
"article with multiple h1",
|
||||
"<html><head></head><body><article><time datetime=\"2004-05-14\">May 14 2004</time><h1>Hello</h1><h1>World</h1>hello world</article></body></html>",
|
||||
[]*time.Time{&testDate},
|
||||
[]string{"<article><time datetime=\"2004-05-14\">May 14 2004</time><h1>Hello</h1><h1>World</h1>hello world</article>"},
|
||||
[]string{"Hello"},
|
||||
&testDate,
|
||||
"<article><time datetime=\"2004-05-14\">May 14 2004</time>hello world</article>",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func (t *testing.T) {
|
||||
html, err := html.Parse(strings.NewReader(tt.input))
|
||||
item := feed.FeedItem{}
|
||||
err := item.ParseContent(tt.input)
|
||||
if err != nil {
|
||||
t.Errorf("error: %s", err)
|
||||
}
|
||||
page := feed.SitePage{
|
||||
Url: "",
|
||||
Title: "",
|
||||
Root: html,
|
||||
Errors: make([]error, 0),
|
||||
}
|
||||
items, err := page.Parse()
|
||||
if err != nil {
|
||||
t.Errorf("error: %s", err)
|
||||
}
|
||||
for i, item := range items {
|
||||
if item.RawText != tt.want_article[i] {
|
||||
t.Errorf("got %s, want %s", item.RawText, tt.want_article[i])
|
||||
}
|
||||
if tt.want_time[i] != nil && !item.PubTime.Equal(*tt.want_time[i]) {
|
||||
t.Errorf("got %s, want %s", item.PubTime, *tt.want_time[i])
|
||||
}
|
||||
if item.Title != tt.want_title[i] {
|
||||
t.Errorf("got %s, want %s", item.Title, tt.want_title[i])
|
||||
if item.RawText != tt.want_article {
|
||||
t.Errorf("got %s, want %s", item.RawText, tt.want_article)
|
||||
}
|
||||
if tt.want_time != nil && !item.PubTime.Equal(*tt.want_time) {
|
||||
t.Errorf("got %s, want %s", item.PubTime, *tt.want_time)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue