dave/jysk-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Replace redis with sqlite

Browse Source
This commit is contained in:
PhatPhuckDave
2024-08-24 16:17:17 +02:00
parent 11dac10072
commit 92dbe446e8
5 changed files with 92 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@@ -1,2 +1,3 @@
.env
main.log
jysk.db

77
db.go Normal file
View File

@@ -0,0 +1,77 @@
package main
import (
"database/sql"
"fmt"
"log"
"os"
"time"
_ "github.com/mattn/go-sqlite3"
)
type DB struct {
Ready bool
path string
readConn *sql.DB
writeConn *sql.DB
}
func (db *DB) Open() error {
if db.path == "" {
return fmt.Errorf("database path not set")
}
file, err := os.Open(db.path)
if err != nil {
if os.IsNotExist(err) {
log.Printf("Database file does not exist at %s, creating", db.path)
file, err := os.Create(db.path)
if err != nil {
return fmt.Errorf("failed to create database file: %v", err)
}
log.Printf("Database created at %s", db.path)
file.Close()
} else {
return fmt.Errorf("failed to open database file: %v", err)
}
}
file.Close()
writeConn, err := sql.Open("sqlite3", db.path+"?_journal=WAL&_synchronous=NORMAL")
if err != nil {
Error.Printf("%++v", err)
return err
}
writeConn.SetMaxOpenConns(1)
writeConn.SetConnMaxIdleTime(30 * time.Second)
writeConn.SetConnMaxLifetime(30 * time.Second)
db.writeConn = writeConn
readConn, err := sql.Open("sqlite3", db.path+"?mode=ro&_journal=WAL&_synchronous=NORMAL&_mode=ro")
if err != nil {
Error.Printf("%++v", err)
return err
}
readConn.SetMaxOpenConns(4)
readConn.SetConnMaxIdleTime(30 * time.Second)
readConn.SetConnMaxLifetime(30 * time.Second)
db.readConn = readConn
db.Ready = true
return nil
}
func (db *DB) Close() error {
err := db.writeConn.Close()
if err != nil {
return err
}
err = db.readConn.Close()
if err != nil {
return err
}
return nil
}

5
go.mod
View File

@@ -4,13 +4,10 @@ go 1.23.0
require (
github.com/PuerkitoBio/goquery v1.9.2
github.com/joho/godotenv v1.5.1
github.com/redis/go-redis/v9 v9.6.1
github.com/mattn/go-sqlite3 v1.14.22
)
require (
github.com/andybalholm/cascadia v1.3.2 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
golang.org/x/net v0.24.0 // indirect
)

14
go.sum
View File

@@ -2,18 +2,8 @@ github.com/PuerkitoBio/goquery v1.9.2 h1:4/wZksC3KgkQw7SQgkKotmKljk0M6V8TUvA8Wb4
github.com/PuerkitoBio/goquery v1.9.2/go.mod h1:GHPCaP0ODyyxqcNoFGYlAprUFH81NuRPd0GX3Zu2Mvk=
github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss=
github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU=
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
github.com/redis/go-redis/v9 v9.6.1 h1:HHDteefn6ZkTtY5fGUE8tj8uy85AHk6zP7CpzIAM0y4=
github.com/redis/go-redis/v9 v9.6.1/go.mod h1:0C0c6ycQsdpVNQpxb1njEQIqkx5UcsM8FJCQLgE9+RA=
github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=

67
main.go
View File

@@ -1,7 +1,6 @@
package main
import (
"context"
"fmt"
"io"
"log"
@@ -9,14 +8,9 @@ import (
"os"
"regexp"
"strconv"
"strings"
"time"
_ "embed"
"github.com/PuerkitoBio/goquery"
"github.com/joho/godotenv"
"github.com/redis/go-redis/v9"
)
var Error *log.Logger
@@ -40,65 +34,22 @@ func init() {
log.Lmicroseconds|log.Lshortfile)
}
//go:embed .env
var env string
var ctx = context.Background()
var rdb *redis.Client
var priceRegex = regexp.MustCompile(`\d+`)
const url = "https://jysk.hr/spavaca-soba/madraci/madraci-s-oprugama/madrac-s-oprugama-140x200-gold-s110-dreamzone"
func main() {
envvar, err := godotenv.Parse(strings.NewReader(env))
db := DB{path: "jysk.db"}
err := db.Open()
if err != nil {
Error.Fatalf("Error parsing .env file: %v", err)
os.Exit(1)
}
redisHost, ok := envvar["REDIS_HOST"]
if !ok {
Error.Fatalf("REDIS_HOST not set in .env file")
return
}
redisPassword, ok := envvar["REDIS_PASSWORD"]
if !ok {
Error.Fatalf("REDIS_PASSWORD not set in .env file")
Error.Fatalf("Error opening database: %v", err)
return
}
defer db.Close()
rdb = redis.NewClient(&redis.Options{
Addr: redisHost,
Password: redisPassword,
DB: 0,
})
// rdb.ZAdd(ctx)
// err = rdb.Set(ctx, "key", "value", 0).Err()
// if err != nil {
// panic(err)
// }
// val, err := rdb.Get(ctx, "key").Result()
// if err != nil {
// panic(err)
// }
// fmt.Println("key", val)
// val2, err := rdb.Get(ctx, "key2").Result()
// if err == redis.Nil {
// fmt.Println("key2 does not exist")
// } else if err != nil {
// panic(err)
// } else {
// fmt.Println("key2", val2)
// }
price, err := GetPrice()
if err != nil {
Error.Fatalf("Error getting price: %v", err)
return
price := Price{
Price: 100,
Timestamp: time.Now(),
}
log.Printf("%#v", price)
}
@@ -142,3 +93,7 @@ func GetPrice() (Price, error) {
return price, nil
}
func SavePrice(price Price) error {
return nil
}