From c398d9204d10fed995fd3ea6916f3576be0e1b9a Mon Sep 17 00:00:00 2001 From: Igor Chubin Date: Sun, 18 Dec 2022 15:44:20 +0100 Subject: [PATCH] Implement location resolution interface --- internal/geo/location/cache.go | 78 +++++++++++++++++++++++++++++- internal/geo/location/location.go | 41 ---------------- internal/geo/location/nominatim.go | 5 +- internal/geo/location/response.go | 41 ++++++++++++++++ internal/geo/location/search.go | 42 ++++++++++++++++ internal/processor/processor.go | 9 ++++ srv.go | 19 ++++++-- 7 files changed, 188 insertions(+), 47 deletions(-) create mode 100644 internal/geo/location/response.go create mode 100644 internal/geo/location/search.go diff --git a/internal/geo/location/cache.go b/internal/geo/location/cache.go index 8530c2c..7aca700 100644 --- a/internal/geo/location/cache.go +++ b/internal/geo/location/cache.go @@ -2,13 +2,16 @@ package location import ( "encoding/json" + "errors" "fmt" "os" "path" + "strconv" "strings" "github.com/samonzeweb/godb" "github.com/samonzeweb/godb/adapters/sqlite" + log "github.com/sirupsen/logrus" "github.com/zsefvlol/timezonemapper" "github.com/chubin/wttr.in/internal/config" @@ -22,6 +25,7 @@ import ( type Cache struct { config *config.Config db *godb.DB + searcher *Searcher indexField string filesCacheDir string } @@ -34,11 +38,14 @@ func NewCache(config *config.Config) (*Cache, error) { ) if config.Geo.LocationCacheType == types.CacheTypeDB { - db, err = godb.Open(sqlite.Adapter, config.Geo.IPCacheDB) + log.Debugln("using db for location cache") + db, err = godb.Open(sqlite.Adapter, config.Geo.LocationCacheDB) if err != nil { return nil, err } + log.Debugln("db file:", config.Geo.LocationCacheDB) + // Needed for "upsert" implementation in Put() db.UseErrorParser() } @@ -48,9 +55,39 @@ func NewCache(config *config.Config) (*Cache, error) { db: db, indexField: "name", filesCacheDir: config.Geo.LocationCache, + searcher: NewSearcher(config), }, nil } +// Resolve returns location information for specified location. +// If the information is found in the cache, it is returned. +// If it is not found, the external service is queried, +// and the result is stored in the cache. +func (c *Cache) Resolve(location string) (*Location, error) { + location = normalizeLocationName(location) + + loc, err := c.Read(location) + if !errors.Is(err, types.ErrNotFound) { + return loc, err + } + + log.Debugln("geo/location: not found in cache:", location) + loc, err = c.searcher.Search(location) + if err != nil { + return nil, err + } + + loc.Name = location + loc.Timezone = latLngToTimezoneString(loc.Lat, loc.Lon) + + err = c.Put(location, loc) + if err != nil { + return nil, err + } + + return loc, nil +} + // Read returns location information from the cache, if found, // or types.ErrNotFound if not found. If the entry is found, but its format // is invalid, types.ErrInvalidCacheEntry is returned. @@ -108,6 +145,11 @@ func (c *Cache) readFromCacheDB(addr string) (*Location, error) { err := c.db.Select(&result). Where(c.indexField+" = ?", addr). Do() + + if strings.Contains(fmt.Sprint(err), "no rows in result set") { + return nil, types.ErrNotFound + } + if err != nil { return nil, err } @@ -116,6 +158,7 @@ func (c *Cache) readFromCacheDB(addr string) (*Location, error) { } func (c *Cache) Put(addr string, loc *Location) error { + log.Infoln("geo/location: storing in cache:", loc) if c.config.Geo.IPCacheType == types.CacheTypeDB { return c.putToCacheDB(loc) } @@ -140,3 +183,36 @@ func (c *Cache) putToCacheFile(addr string, loc fmt.Stringer) error { func (c *Cache) cacheFile(item string) string { return path.Join(c.filesCacheDir, item) } + +// normalizeLocationName converts name into the standard location form +// with the following steps: +// - remove excessive spaces, +// - remove quotes, +// - convert to lover case. +func normalizeLocationName(name string) string { + name = strings.ReplaceAll(name, `"`, " ") + name = strings.ReplaceAll(name, `'`, " ") + name = strings.TrimSpace(name) + name = strings.Join(strings.Fields(name), " ") + + return strings.ToLower(name) +} + +// latLngToTimezoneString returns timezone for lat, lon, +// or an empty string if they are invalid. +func latLngToTimezoneString(lat, lon string) string { + latFloat, err := strconv.ParseFloat(lat, 64) + if err != nil { + log.Errorln("geoloc: latLngToTimezoneString:", err) + + return "" + } + lonFloat, err := strconv.ParseFloat(lon, 64) + if err != nil { + log.Errorln("geoloc: latLngToTimezoneString:", err) + + return "" + } + + return timezonemapper.LatLngToTimezoneString(latFloat, lonFloat) +} diff --git a/internal/geo/location/location.go b/internal/geo/location/location.go index f33b20d..0338b40 100644 --- a/internal/geo/location/location.go +++ b/internal/geo/location/location.go @@ -3,8 +3,6 @@ package location import ( "encoding/json" "log" - - "github.com/chubin/wttr.in/internal/config" ) type Location struct { @@ -26,42 +24,3 @@ func (l *Location) String() string { return string(bytes) } - -type Provider interface { - Query(location string) (*Location, error) -} - -type Searcher struct { - providers []Provider -} - -// NewSearcher returns a new Searcher for the specified config. -func NewSearcher(config *config.Config) *Searcher { - providers := []Provider{} - for _, p := range config.Geo.Nominatim { - providers = append(providers, NewNominatim(p.Name, p.URL, p.Token)) - } - - return &Searcher{ - providers: providers, - } -} - -// Search makes queries through all known providers, -// and returns response, as soon as it is not nil. -// If all responses were nil, the last response is returned. -func (s *Searcher) Search(location string) (*Location, error) { - var ( - err error - result *Location - ) - - for _, p := range s.providers { - result, err = p.Query(location) - if result != nil && err == nil { - return result, nil - } - } - - return result, err -} diff --git a/internal/geo/location/nominatim.go b/internal/geo/location/nominatim.go index e16eef7..6b3e752 100644 --- a/internal/geo/location/nominatim.go +++ b/internal/geo/location/nominatim.go @@ -4,11 +4,11 @@ import ( "encoding/json" "fmt" "io/ioutil" - "log" "net/http" "net/url" "github.com/chubin/wttr.in/internal/types" + log "github.com/sirupsen/logrus" ) type Nominatim struct { @@ -38,7 +38,7 @@ func (n *Nominatim) Query(location string) (*Location, error) { "%s?q=%s&format=json&accept-language=native&limit=1&key=%s", n.url, url.QueryEscape(location), n.token) - log.Println(urlws) + log.Debugln("nominatim:", urlws) resp, err := http.Get(urlws) if err != nil { return nil, fmt.Errorf("%s: %w", n.name, err) @@ -55,6 +55,7 @@ func (n *Nominatim) Query(location string) (*Location, error) { return nil, fmt.Errorf("%w: %s: %s", types.ErrUpstream, n.name, errResponse.Error) } + log.Debugln("nominatim: response: ", string(body)) err = json.Unmarshal(body, &result) if err != nil { return nil, fmt.Errorf("%s: %w", n.name, err) diff --git a/internal/geo/location/response.go b/internal/geo/location/response.go new file mode 100644 index 0000000..6ebc6fa --- /dev/null +++ b/internal/geo/location/response.go @@ -0,0 +1,41 @@ +package location + +import ( + "encoding/json" + "fmt" + "net/http" + + "github.com/chubin/wttr.in/internal/routing" +) + +// Response provides routing interface to the geo cache. +func (c *Cache) Response(r *http.Request) *routing.Cadre { + var ( + locationName = r.URL.Query().Get("location") + loc *Location + bytes []byte + err error + ) + + if locationName == "" { + return errorResponse("location is not specified") + } + + loc, err = c.Resolve(locationName) + if err != nil { + return errorResponse(fmt.Sprint(err)) + } + + bytes, err = json.Marshal(loc) + if err != nil { + return errorResponse(fmt.Sprint(err)) + } + + return &routing.Cadre{Body: bytes} +} + +func errorResponse(s string) *routing.Cadre { + return &routing.Cadre{Body: []byte( + fmt.Sprintf(`{"error": %q}`, s), + )} +} diff --git a/internal/geo/location/search.go b/internal/geo/location/search.go new file mode 100644 index 0000000..c05cdb0 --- /dev/null +++ b/internal/geo/location/search.go @@ -0,0 +1,42 @@ +package location + +import "github.com/chubin/wttr.in/internal/config" + +type Provider interface { + Query(location string) (*Location, error) +} + +type Searcher struct { + providers []Provider +} + +// NewSearcher returns a new Searcher for the specified config. +func NewSearcher(config *config.Config) *Searcher { + providers := []Provider{} + for _, p := range config.Geo.Nominatim { + providers = append(providers, NewNominatim(p.Name, p.URL, p.Token)) + } + + return &Searcher{ + providers: providers, + } +} + +// Search makes queries through all known providers, +// and returns response, as soon as it is not nil. +// If all responses were nil, the last response is returned. +func (s *Searcher) Search(location string) (*Location, error) { + var ( + err error + result *Location + ) + + for _, p := range s.providers { + result, err = p.Query(location) + if result != nil && err == nil { + return result, nil + } + } + + return result, err +} diff --git a/internal/processor/processor.go b/internal/processor/processor.go index 5623c5c..90abea7 100644 --- a/internal/processor/processor.go +++ b/internal/processor/processor.go @@ -16,6 +16,7 @@ import ( "github.com/chubin/wttr.in/internal/config" geoip "github.com/chubin/wttr.in/internal/geo/ip" + geoloc "github.com/chubin/wttr.in/internal/geo/location" "github.com/chubin/wttr.in/internal/routing" "github.com/chubin/wttr.in/internal/stats" "github.com/chubin/wttr.in/internal/util" @@ -58,6 +59,7 @@ type RequestProcessor struct { upstreamTransport *http.Transport config *config.Config geoIPCache *geoip.Cache + geoLocation *geoloc.Cache } // NewRequestProcessor returns new RequestProcessor. @@ -84,18 +86,25 @@ func NewRequestProcessor(config *config.Config) (*RequestProcessor, error) { return nil, err } + geoLocation, err := geoloc.NewCache(config) + if err != nil { + return nil, err + } + rp := &RequestProcessor{ lruCache: lruCache, stats: stats.New(), upstreamTransport: transport, config: config, geoIPCache: geoCache, + geoLocation: geoLocation, } // Initialize routes. rp.router.AddPath("/:stats", rp.stats) rp.router.AddPath("/:geo-ip-get", rp.geoIPCache) rp.router.AddPath("/:geo-ip-put", rp.geoIPCache) + rp.router.AddPath("/:geo-location", rp.geoLocation) return rp, nil } diff --git a/srv.go b/srv.go index 25c11a1..62afa87 100644 --- a/srv.go +++ b/srv.go @@ -4,11 +4,12 @@ import ( "crypto/tls" "fmt" "io" - "log" + stdlog "log" "net/http" "time" "github.com/alecthomas/kong" + log "github.com/sirupsen/logrus" "github.com/chubin/wttr.in/internal/config" geoip "github.com/chubin/wttr.in/internal/geo/ip" @@ -27,6 +28,7 @@ var cli struct { ConvertGeoIPCache bool `name:"convert-geo-ip-cache" help:"Convert Geo IP data cache to SQlite"` ConvertGeoLocationCache bool `name:"convert-geo-location-cache" help:"Convert Geo Location data cache to SQlite"` GeoResolve string `name:"geo-resolve" help:"Resolve location"` + LogLevel string `name:"log-level" short:"l" help:"Show log messages with level" default:"info"` } const logLineStart = "LOG_LINE_START " @@ -42,7 +44,7 @@ func copyHeader(dst, src http.Header) { func serveHTTP(mux *http.ServeMux, port int, logFile io.Writer, errs chan<- error) { srv := &http.Server{ Addr: fmt.Sprintf(":%d", port), - ErrorLog: log.New(logFile, logLineStart, log.LstdFlags), + ErrorLog: stdlog.New(logFile, logLineStart, stdlog.LstdFlags), ReadTimeout: 5 * time.Second, WriteTimeout: 10 * time.Second, IdleTimeout: 1 * time.Second, @@ -63,7 +65,7 @@ func serveHTTPS(mux *http.ServeMux, port int, certFile, keyFile string, logFile } srv := &http.Server{ Addr: fmt.Sprintf(":%d", port), - ErrorLog: log.New(logFile, logLineStart, log.LstdFlags), + ErrorLog: stdlog.New(logFile, logLineStart, stdlog.LstdFlags), ReadTimeout: 5 * time.Second, WriteTimeout: 20 * time.Second, IdleTimeout: 1 * time.Second, @@ -173,6 +175,7 @@ func main() { ) ctx := kong.Parse(&cli) + ctx.FatalIfErrorf(setLogLevel(cli.LogLevel)) if cli.ConfigFile != "" { conf, err = config.Load(cli.ConfigFile) @@ -230,3 +233,13 @@ func convertGeoLocationCache(conf *config.Config) error { return geoLocCache.ConvertCache() } + +func setLogLevel(logLevel string) error { + parsedLevel, err := log.ParseLevel(logLevel) + if err != nil { + return err + } + log.SetLevel(parsedLevel) + + return nil +}