Files
jysk-scraper/main.go

145 lines
3.1 KiB
Go

package main
import (
"context"
"fmt"
"io"
"log"
"net/http"
"os"
"regexp"
"strconv"
"strings"
"time"
_ "embed"
"github.com/PuerkitoBio/goquery"
"github.com/joho/godotenv"
"github.com/redis/go-redis/v9"
)
var Error *log.Logger
var Warning *log.Logger
func init() {
log.SetFlags(log.Lmicroseconds | log.Lshortfile)
logFile, err := os.Create("main.log")
if err != nil {
log.Printf("Error creating log file: %v", err)
return
}
logger := io.MultiWriter(os.Stdout, logFile)
log.SetOutput(logger)
Error = log.New(io.MultiWriter(logFile, os.Stderr, os.Stdout),
fmt.Sprintf("%sERROR:%s ", "\033[0;101m", "\033[0m"),
log.Lmicroseconds|log.Lshortfile)
Warning = log.New(io.MultiWriter(logFile, os.Stdout),
fmt.Sprintf("%sWarning:%s ", "\033[0;93m", "\033[0m"),
log.Lmicroseconds|log.Lshortfile)
}
//go:embed .env
var env string
var ctx = context.Background()
var rdb *redis.Client
var priceRegex = regexp.MustCompile(`\d+`)
const url = "https://jysk.hr/spavaca-soba/madraci/madraci-s-oprugama/madrac-s-oprugama-140x200-gold-s110-dreamzone"
func main() {
envvar, err := godotenv.Parse(strings.NewReader(env))
if err != nil {
Error.Fatalf("Error parsing .env file: %v", err)
os.Exit(1)
}
redisHost, ok := envvar["REDIS_HOST"]
if !ok {
Error.Fatalf("REDIS_HOST not set in .env file")
return
}
redisPassword, ok := envvar["REDIS_PASSWORD"]
if !ok {
Error.Fatalf("REDIS_PASSWORD not set in .env file")
return
}
rdb = redis.NewClient(&redis.Options{
Addr: redisHost,
Password: redisPassword,
DB: 0,
})
// rdb.ZAdd(ctx)
// err = rdb.Set(ctx, "key", "value", 0).Err()
// if err != nil {
// panic(err)
// }
// val, err := rdb.Get(ctx, "key").Result()
// if err != nil {
// panic(err)
// }
// fmt.Println("key", val)
// val2, err := rdb.Get(ctx, "key2").Result()
// if err == redis.Nil {
// fmt.Println("key2 does not exist")
// } else if err != nil {
// panic(err)
// } else {
// fmt.Println("key2", val2)
// }
price, err := GetPrice()
if err != nil {
Error.Fatalf("Error getting price: %v", err)
return
}
log.Printf("%#v", price)
}
type Price struct {
Price int
Timestamp time.Time
}
func GetPrice() (Price, error) {
price := Price{Timestamp: time.Now()}
res, err := http.Get(url)
if err != nil {
return price, fmt.Errorf("failed querying url %s with error %w", url, err)
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
return price, fmt.Errorf("failed getting data from %s with status: %s", url, res.Status)
}
doc, err := goquery.NewDocumentFromReader(res.Body)
if err != nil {
log.Fatal(err)
}
doc.Find("span.ssr-product-price__value").Each(func(i int, s *goquery.Selection) {
log.Printf("Found price element with text: %s", s.Text())
res := priceRegex.FindStringSubmatch(s.Text())
if len(res) != 1 {
Error.Fatalf("failed parsing price: %q", res)
return
}
price.Price, err = strconv.Atoi(res[0])
if err != nil {
Error.Fatalf("failed converting price to int: %v", err)
return
}
})
return price, nil
}