From b62a7a7a28b9a067254b56af3f419a6c7aa271d2 Mon Sep 17 00:00:00 2001 From: Stephen Searles Date: Thu, 7 Sep 2017 18:34:05 -0500 Subject: [PATCH 1/5] moving media code to a subpackage --- caddyhugo.go | 3 +- media.go | 246 +------------------------- media/media.go | 252 +++++++++++++++++++++++++++ media_test.go => media/media_test.go | 4 +- setup.go | 3 +- 5 files changed, 261 insertions(+), 247 deletions(-) create mode 100644 media/media.go rename media_test.go => media/media_test.go (93%) diff --git a/caddyhugo.go b/caddyhugo.go index 89c227f..f7c53bf 100644 --- a/caddyhugo.go +++ b/caddyhugo.go @@ -11,6 +11,7 @@ import ( "git.stephensearles.com/stephen/acedoc" "git.stephensearles.com/stephen/caddy-hugo2/comments" + "git.stephensearles.com/stephen/caddy-hugo2/media" "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/hugolib" @@ -35,7 +36,7 @@ type CaddyHugo struct { Dir string - Media *MediaSource + Media *media.MediaSource Comments *comments.Service docs map[string]*editSession diff --git a/media.go b/media.go index 5a9ceba..c33570d 100644 --- a/media.go +++ b/media.go @@ -3,206 +3,14 @@ package caddyhugo import ( "fmt" "image" - _ "image/gif" // for processing images - "image/jpeg" - _ "image/png" // for processing images "io" "net/http" - "os" "path" - "path/filepath" - "regexp" - "strconv" "strings" - "github.com/nfnt/resize" + "git.stephensearles.com/stephen/caddy-hugo2/media" ) -type MediaSource struct { - StorageDir string - ThumbDir string -} - -func (ms *MediaSource) LocationOrig(m Media) string { - return path.Join(ms.StorageDir, m.Name) -} - -func (ms *MediaSource) ThumbPath(m Media, size image.Rectangle) string { - w := size.Dx() - h := size.Dy() - - var ws, hs string - - if w != 0 { - ws = fmt.Sprint(w) - } - if h != 0 { - hs = fmt.Sprint(h) - } - - thumbSlug := filepath.Join(fmt.Sprintf("%sx%s", ws, hs), m.Name) - return path.Join("/media", thumbSlug) -} - -func (ms *MediaSource) ThumbFilename(m Media, size image.Rectangle) string { - return filepath.Join(ms.ThumbDir, ms.ThumbPath(m, size)) -} - -func (ms *MediaSource) receiveNewMedia(name string, r io.Reader) error { - dest := path.Join(ms.StorageDir, name) - f, err := os.OpenFile(dest, os.O_WRONLY|os.O_CREATE, 0644) - if err != nil { - return err - } - - _, err = io.Copy(f, r) - if err != nil { - return err - } - - return f.Close() -} - -type Media struct { - Type string - Name string - Size image.Rectangle -} - -func (ms *MediaSource) Size(name string) (image.Rectangle, error) { - f, err := os.Open(name) - if err != nil { - return image.ZR, err - } - defer f.Close() - - cfg, _, err := image.DecodeConfig(f) - if err != nil { - return image.ZR, err - } - - width := cfg.Width - height := cfg.Height - - return image.Rect(0, 0, width, height), nil -} - -func (ms *MediaSource) ThumbMax(m Media, maxDim int) (string, image.Rectangle, error) { - f, err := os.Open(ms.LocationOrig(m)) - if err != nil { - return "", image.ZR, err - } - defer f.Close() - - cfg, _, err := image.DecodeConfig(f) - if err != nil { - return "", image.ZR, err - } - - width := cfg.Width - height := cfg.Height - - if width > height { - height = height * maxDim / width - width = maxDim - } else { - width = width * maxDim / height - height = maxDim - } - - size := image.Rect(0, 0, width, height) - if ms.HasThumb(m, size) { - return ms.ThumbPath(m, size), size, nil - } - - _, err = f.Seek(0, io.SeekStart) - if err != nil { - return "", image.ZR, err - } - - img, _, err := image.Decode(f) - if err != nil { - return "", image.ZR, err - } - - src, err := ms.ThumbImage(img, m, size) - return src, size, err -} - -func (ms *MediaSource) HasThumb(m Media, size image.Rectangle) bool { - _, err := os.Stat(ms.ThumbFilename(m, size)) - return err == nil -} - -func (ms *MediaSource) ByName(name string) *Media { - size, _ := ms.Size(path.Join(ms.StorageDir, name)) - return &Media{ - Type: "image", - Name: name, - Size: size, - } -} - -func (ms *MediaSource) Thumb(m Media, size image.Rectangle) (string, error) { - if ms.HasThumb(m, size) { - return ms.ThumbPath(m, size), nil - } - - f, err := os.Open(ms.LocationOrig(m)) - if err != nil { - return "", err - } - defer f.Close() - - img, _, err := image.Decode(f) - if err != nil { - return "", err - } - - return ms.ThumbImage(img, m, size) -} - -func (ms *MediaSource) ThumbImage(img image.Image, m Media, size image.Rectangle) (string, error) { - - thumbLoc := ms.ThumbFilename(m, size) - os.MkdirAll(path.Dir(thumbLoc), 0755) - fthumb, err := os.OpenFile(thumbLoc, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0655) - if err != nil { - return "", err - } - - img = resize.Resize(uint(size.Dx()), uint(size.Dy()), img, resize.Bilinear) - - err = jpeg.Encode(fthumb, img, nil) - if err != nil { - return "", err - } - - err = fthumb.Close() - if err != nil { - return "", err - } - - return ms.ThumbPath(m, size), nil -} - -func (ms *MediaSource) Walk() ([]*Media, error) { - var media []*Media - - err := filepath.Walk(ms.StorageDir, func(name string, fi os.FileInfo, err error) error { - if err != nil { - return err - } - if fi.IsDir() { - return nil - } - media = append(media, ms.ByName(path.Base(name))) - return nil - }) - - return media, err -} - func (ch *CaddyHugo) uploadMedia(w http.ResponseWriter, r *http.Request) (int, error) { if ch.Media == nil { http.NotFound(w, r) @@ -227,7 +35,7 @@ func (ch *CaddyHugo) uploadMedia(w http.ResponseWriter, r *http.Request) (int, e name := part.FileName() if name != "" { - err = ch.Media.receiveNewMedia(name, part) + err = ch.Media.ReceiveNewMedia(name, part) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return 500, nil @@ -314,49 +122,6 @@ func (ch *CaddyHugo) serveMediaPage(w http.ResponseWriter, r *http.Request) (int return 200, nil } -var ( - sizeString = regexp.MustCompile(`([0-9]*)(x)?([0-9]*)`) -) - -func parseSizeString(str string, actual image.Rectangle) (image.Rectangle, error) { - var err = fmt.Errorf("expected a size string {width}x{height}, saw %q", str) - - strs := sizeString.FindStringSubmatch(str) - if len(strs) < 4 { - return image.ZR, err - } - - var w, h int - var strconvErr error - - if strs[1] != "" { - w, strconvErr = strconv.Atoi(strs[1]) - if strconvErr != nil { - return image.ZR, err - } - } - - if strs[3] != "" { - h, strconvErr = strconv.Atoi(strs[3]) - if strconvErr != nil { - return image.ZR, err - } - } - - if strs[2] != "x" { - // w was the only dimension given, so set it to the greater dimension - // of the actual image size - if actual.Dx() > actual.Dy() { - h = 0 - } else { - h = w - w = 0 - } - } - - return image.Rect(0, 0, w, h), nil -} - func (ch *CaddyHugo) serveMedia(w http.ResponseWriter, r *http.Request) (int, error) { if ch.Media == nil { http.NotFound(w, r) @@ -372,7 +137,7 @@ func (ch *CaddyHugo) serveMedia(w http.ResponseWriter, r *http.Request) (int, er if len(segs) >= 4 && len(segs) > 2 { var err error - size, err = parseSizeString(segs[len(segs)-2], m.Size) + size, err = media.ParseSizeString(segs[len(segs)-2], m.Size) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return 400, nil @@ -395,8 +160,3 @@ func (ch *CaddyHugo) serveMedia(w http.ResponseWriter, r *http.Request) (int, er return 200, nil } - -func removeExtension(name string) string { - ext := path.Ext(name) - return name[:len(name)-len(ext)] -} diff --git a/media/media.go b/media/media.go new file mode 100644 index 0000000..16ae2e2 --- /dev/null +++ b/media/media.go @@ -0,0 +1,252 @@ +package media + +import ( + "fmt" + "image" + "image/jpeg" + "io" + "os" + "path" + "path/filepath" + "regexp" + "strconv" + + // for processing images + _ "image/gif" + _ "image/png" + + "github.com/nfnt/resize" +) + +type MediaSource struct { + StorageDir string + ThumbDir string +} + +func (ms *MediaSource) LocationOrig(m Media) string { + return path.Join(ms.StorageDir, m.Name) +} + +func (ms *MediaSource) ThumbPath(m Media, size image.Rectangle) string { + w := size.Dx() + h := size.Dy() + + var ws, hs string + + if w != 0 { + ws = fmt.Sprint(w) + } + if h != 0 { + hs = fmt.Sprint(h) + } + + thumbSlug := filepath.Join(fmt.Sprintf("%sx%s", ws, hs), m.Name) + return path.Join("/media", thumbSlug) +} + +func (ms *MediaSource) ThumbFilename(m Media, size image.Rectangle) string { + return filepath.Join(ms.ThumbDir, ms.ThumbPath(m, size)) +} + +func (ms *MediaSource) ReceiveNewMedia(name string, r io.Reader) error { + dest := path.Join(ms.StorageDir, name) + f, err := os.OpenFile(dest, os.O_WRONLY|os.O_CREATE, 0644) + if err != nil { + return err + } + + _, err = io.Copy(f, r) + if err != nil { + return err + } + + return f.Close() +} + +type Media struct { + Type string + Name string + Size image.Rectangle +} + +func (ms *MediaSource) Size(name string) (image.Rectangle, error) { + f, err := os.Open(name) + if err != nil { + return image.ZR, err + } + defer f.Close() + + cfg, _, err := image.DecodeConfig(f) + if err != nil { + return image.ZR, err + } + + width := cfg.Width + height := cfg.Height + + return image.Rect(0, 0, width, height), nil +} + +func (ms *MediaSource) ThumbMax(m Media, maxDim int) (string, image.Rectangle, error) { + f, err := os.Open(ms.LocationOrig(m)) + if err != nil { + return "", image.ZR, err + } + defer f.Close() + + cfg, _, err := image.DecodeConfig(f) + if err != nil { + return "", image.ZR, err + } + + width := cfg.Width + height := cfg.Height + + if width > height { + height = height * maxDim / width + width = maxDim + } else { + width = width * maxDim / height + height = maxDim + } + + size := image.Rect(0, 0, width, height) + if ms.HasThumb(m, size) { + return ms.ThumbPath(m, size), size, nil + } + + _, err = f.Seek(0, io.SeekStart) + if err != nil { + return "", image.ZR, err + } + + img, _, err := image.Decode(f) + if err != nil { + return "", image.ZR, err + } + + src, err := ms.ThumbImage(img, m, size) + return src, size, err +} + +func (ms *MediaSource) HasThumb(m Media, size image.Rectangle) bool { + _, err := os.Stat(ms.ThumbFilename(m, size)) + return err == nil +} + +func (ms *MediaSource) ByName(name string) *Media { + size, _ := ms.Size(path.Join(ms.StorageDir, name)) + return &Media{ + Type: "image", + Name: name, + Size: size, + } +} + +func (ms *MediaSource) Thumb(m Media, size image.Rectangle) (string, error) { + if ms.HasThumb(m, size) { + return ms.ThumbPath(m, size), nil + } + + f, err := os.Open(ms.LocationOrig(m)) + if err != nil { + return "", err + } + defer f.Close() + + img, _, err := image.Decode(f) + if err != nil { + return "", err + } + + return ms.ThumbImage(img, m, size) +} + +func (ms *MediaSource) ThumbImage(img image.Image, m Media, size image.Rectangle) (string, error) { + + thumbLoc := ms.ThumbFilename(m, size) + os.MkdirAll(path.Dir(thumbLoc), 0755) + fthumb, err := os.OpenFile(thumbLoc, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0655) + if err != nil { + return "", err + } + + img = resize.Resize(uint(size.Dx()), uint(size.Dy()), img, resize.Bilinear) + + err = jpeg.Encode(fthumb, img, nil) + if err != nil { + return "", err + } + + err = fthumb.Close() + if err != nil { + return "", err + } + + return ms.ThumbPath(m, size), nil +} + +func (ms *MediaSource) Walk() ([]*Media, error) { + var media []*Media + + err := filepath.Walk(ms.StorageDir, func(name string, fi os.FileInfo, err error) error { + if err != nil { + return err + } + if fi.IsDir() { + return nil + } + media = append(media, ms.ByName(path.Base(name))) + return nil + }) + + return media, err +} + +var ( + sizeString = regexp.MustCompile(`([0-9]*)(x)?([0-9]*)`) +) + +func ParseSizeString(str string, actual image.Rectangle) (image.Rectangle, error) { + var err = fmt.Errorf("expected a size string {width}x{height}, saw %q", str) + + strs := sizeString.FindStringSubmatch(str) + if len(strs) < 4 { + return image.ZR, err + } + + var w, h int + var strconvErr error + + if strs[1] != "" { + w, strconvErr = strconv.Atoi(strs[1]) + if strconvErr != nil { + return image.ZR, err + } + } + + if strs[3] != "" { + h, strconvErr = strconv.Atoi(strs[3]) + if strconvErr != nil { + return image.ZR, err + } + } + + if strs[2] != "x" { + // w was the only dimension given, so set it to the greater dimension + // of the actual image size + if actual.Dx() > actual.Dy() { + h = 0 + } else { + h = w + w = 0 + } + } + + return image.Rect(0, 0, w, h), nil +} + +func removeExtension(name string) string { + ext := path.Ext(name) + return name[:len(name)-len(ext)] +} diff --git a/media_test.go b/media/media_test.go similarity index 93% rename from media_test.go rename to media/media_test.go index 6a91fb0..ace3329 100644 --- a/media_test.go +++ b/media/media_test.go @@ -1,4 +1,4 @@ -package caddyhugo +package media import ( "fmt" @@ -33,7 +33,7 @@ func TestThumbSizeStrings(t *testing.T) { for _, c := range cases { t.Run(fmt.Sprint(c.input, "=>", c.actualSize), func(t *testing.T) { - got, err := parseSizeString(c.input, image.Rect(0, 0, 100, 100)) + got, err := ParseSizeString(c.input, image.Rect(0, 0, 100, 100)) if err != nil { t.Errorf("error parsing size string: %v", err) return diff --git a/setup.go b/setup.go index cad1fda..1623b62 100644 --- a/setup.go +++ b/setup.go @@ -8,6 +8,7 @@ import ( "path" "git.stephensearles.com/stephen/caddy-hugo2/comments" + "git.stephensearles.com/stephen/caddy-hugo2/media" "github.com/gohugoio/hugo/deps" "github.com/gohugoio/hugo/hugofs" @@ -94,7 +95,7 @@ func (ch *CaddyHugo) Setup(dir string) error { return fmt.Errorf("couldn't initialize media: %v", err) } - ch.Media = &MediaSource{ + ch.Media = &media.MediaSource{ StorageDir: path.Join(dir, "media"), ThumbDir: thumbDir, } From 450923be2fe0970ffe3cd7ae02fa7980d60b984a Mon Sep 17 00:00:00 2001 From: Stephen Searles Date: Thu, 7 Sep 2017 19:11:00 -0500 Subject: [PATCH 2/5] sorting images by date; acquiring metadata --- media.go | 4 +-- media/media.go | 95 ++++++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 83 insertions(+), 16 deletions(-) diff --git a/media.go b/media.go index c33570d..c63e293 100644 --- a/media.go +++ b/media.go @@ -77,13 +77,13 @@ func (ch *CaddyHugo) serveMediaPage(w http.ResponseWriter, r *http.Request) (int
`) if ch.Media != nil { - media, err := ch.Media.Walk() + mm, err := ch.Media.Walk() if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return 500, nil } - for _, m := range media { + for _, m := range media.Set(mm).ByDate() { src, size, err := ch.Media.ThumbMax(*m, 100) if err != nil { diff --git a/media/media.go b/media/media.go index 16ae2e2..0674c3c 100644 --- a/media/media.go +++ b/media/media.go @@ -9,18 +9,23 @@ import ( "path" "path/filepath" "regexp" + "sort" "strconv" + "time" // for processing images _ "image/gif" _ "image/png" "github.com/nfnt/resize" + "github.com/tajtiattila/metadata" ) type MediaSource struct { StorageDir string ThumbDir string + + set Set } func (ms *MediaSource) LocationOrig(m Media) string { @@ -49,6 +54,8 @@ func (ms *MediaSource) ThumbFilename(m Media, size image.Rectangle) string { } func (ms *MediaSource) ReceiveNewMedia(name string, r io.Reader) error { + ms.set = nil + dest := path.Join(ms.StorageDir, name) f, err := os.OpenFile(dest, os.O_WRONLY|os.O_CREATE, 0644) if err != nil { @@ -64,9 +71,50 @@ func (ms *MediaSource) ReceiveNewMedia(name string, r io.Reader) error { } type Media struct { - Type string - Name string - Size image.Rectangle + Type string + Name string + Size image.Rectangle + FullName string + + metadata *metadata.Metadata +} + +func (m *Media) Date() time.Time { + m.getMetadata() + + if m.metadata != nil { + raw := m.metadata.Get(metadata.DateTimeOriginal) + if raw != "" { + d, err := time.Parse("2006-01-02T15:04:05", raw) + if err == nil { + return d + } + } + } + + fi, err := os.Stat(m.FullName) + if err == nil { + return fi.ModTime() + } + + return time.Time{} +} + +func (m *Media) getMetadata() error { + if m.metadata != nil { + return nil + } + + f, err := os.Open(m.FullName) + if err != nil { + return err + } + md, err := metadata.Parse(f) + if err != nil { + return err + } + m.metadata = md + return nil } func (ms *MediaSource) Size(name string) (image.Rectangle, error) { @@ -136,11 +184,13 @@ func (ms *MediaSource) HasThumb(m Media, size image.Rectangle) bool { func (ms *MediaSource) ByName(name string) *Media { size, _ := ms.Size(path.Join(ms.StorageDir, name)) - return &Media{ + m := Media{ Type: "image", Name: name, Size: size, } + m.FullName = ms.LocationOrig(m) + return &m } func (ms *MediaSource) Thumb(m Media, size image.Rectangle) (string, error) { @@ -187,20 +237,37 @@ func (ms *MediaSource) ThumbImage(img image.Image, m Media, size image.Rectangle } func (ms *MediaSource) Walk() ([]*Media, error) { - var media []*Media + if ms.set == nil { + var media []*Media + + err := filepath.Walk(ms.StorageDir, func(name string, fi os.FileInfo, err error) error { + if err != nil { + return err + } + if fi.IsDir() { + return nil + } + media = append(media, ms.ByName(path.Base(name))) + return nil + }) - err := filepath.Walk(ms.StorageDir, func(name string, fi os.FileInfo, err error) error { if err != nil { - return err - } - if fi.IsDir() { - return nil + return media, err } - media = append(media, ms.ByName(path.Base(name))) - return nil - }) - return media, err + ms.set = media + } + + return ms.set, nil +} + +type Set []*Media + +func (s Set) ByDate() Set { + sort.Slice(s, func(i, j int) bool { + return s[i].Date().After(s[j].Date()) + }) + return s } var ( From df18b290879ea7282ead686b0fc0b0069a1551e0 Mon Sep 17 00:00:00 2001 From: Stephen Searles Date: Thu, 7 Sep 2017 19:33:42 -0500 Subject: [PATCH 3/5] verifying thumbs now --- media/media.go | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/media/media.go b/media/media.go index 0674c3c..0051ff5 100644 --- a/media/media.go +++ b/media/media.go @@ -28,6 +28,15 @@ type MediaSource struct { set Set } +type Media struct { + Type string + Name string + Size image.Rectangle + FullName string + + metadata *metadata.Metadata +} + func (ms *MediaSource) LocationOrig(m Media) string { return path.Join(ms.StorageDir, m.Name) } @@ -70,15 +79,6 @@ func (ms *MediaSource) ReceiveNewMedia(name string, r io.Reader) error { return f.Close() } -type Media struct { - Type string - Name string - Size image.Rectangle - FullName string - - metadata *metadata.Metadata -} - func (m *Media) Date() time.Time { m.getMetadata() @@ -178,8 +178,11 @@ func (ms *MediaSource) ThumbMax(m Media, maxDim int) (string, image.Rectangle, e } func (ms *MediaSource) HasThumb(m Media, size image.Rectangle) bool { - _, err := os.Stat(ms.ThumbFilename(m, size)) - return err == nil + fi, err := os.Stat(ms.ThumbFilename(m, size)) + if err != nil { + return false + } + return m.Date().Before(fi.ModTime()) } func (ms *MediaSource) ByName(name string) *Media { From 6d479bb9253c3746a026691fb8a6fb63988d94ea Mon Sep 17 00:00:00 2001 From: Stephen Searles Date: Thu, 7 Sep 2017 19:40:41 -0500 Subject: [PATCH 4/5] better reusing media methods --- media/media.go | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/media/media.go b/media/media.go index 0051ff5..1cd74a4 100644 --- a/media/media.go +++ b/media/media.go @@ -168,12 +168,7 @@ func (ms *MediaSource) ThumbMax(m Media, maxDim int) (string, image.Rectangle, e return "", image.ZR, err } - img, _, err := image.Decode(f) - if err != nil { - return "", image.ZR, err - } - - src, err := ms.ThumbImage(img, m, size) + src, err := ms.thumbReader(f, m, size) return src, size, err } @@ -207,7 +202,11 @@ func (ms *MediaSource) Thumb(m Media, size image.Rectangle) (string, error) { } defer f.Close() - img, _, err := image.Decode(f) + return ms.thumbReader(f, m, size) +} + +func (ms *MediaSource) thumbReader(r io.Reader, m Media, size image.Rectangle) (string, error) { + img, _, err := image.Decode(r) if err != nil { return "", err } @@ -216,7 +215,6 @@ func (ms *MediaSource) Thumb(m Media, size image.Rectangle) (string, error) { } func (ms *MediaSource) ThumbImage(img image.Image, m Media, size image.Rectangle) (string, error) { - thumbLoc := ms.ThumbFilename(m, size) os.MkdirAll(path.Dir(thumbLoc), 0755) fthumb, err := os.OpenFile(thumbLoc, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0655) From eb3277598609c3c15e04f873c670636f8bf55a6c Mon Sep 17 00:00:00 2001 From: Stephen Searles Date: Thu, 7 Sep 2017 19:46:09 -0500 Subject: [PATCH 5/5] using a non-tmp dir for thumbs --- setup.go | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/setup.go b/setup.go index 1623b62..8505e1d 100644 --- a/setup.go +++ b/setup.go @@ -3,7 +3,6 @@ package caddyhugo import ( "fmt" "html/template" - "io/ioutil" "os" "path" @@ -90,14 +89,9 @@ func (ch *CaddyHugo) Setup(dir string) error { return fmt.Errorf("edit template invalid: %v", err) } - thumbDir, err := ioutil.TempDir("", "thumbs") - if err != nil { - return fmt.Errorf("couldn't initialize media: %v", err) - } - ch.Media = &media.MediaSource{ StorageDir: path.Join(dir, "media"), - ThumbDir: thumbDir, + ThumbDir: path.Join(dir, "thumbs"), } err = os.MkdirAll(ch.Media.StorageDir, 0755)