Compare commits
32 Commits
Author | SHA1 | Date |
---|---|---|
mappu | 69a48a82b9 | |
mappu | 378d5117f6 | |
mappu | 79a3f7ea1d | |
mappu | 2f92eaac1b | |
mappu | a041f0ba5c | |
mappu | d5f3307f01 | |
mappu | bb400ed6ae | |
mappu | 143c335818 | |
mappu | 161914c34c | |
mappu | 92e0c58b22 | |
mappu | beaa2dc5fe | |
mappu | 136f840905 | |
mappu | e2272bfa0a | |
mappu | 6cfd09fce0 | |
mappu | a91911f351 | |
mappu | 55dba76952 | |
mappu | ac3f1de391 | |
mappu | 2dcf1dbc15 | |
mappu | ae7119f65a | |
mappu | 774de75d6f | |
mappu | 1659609ae1 | |
mappu | dcd3061482 | |
mappu | 448be1bb98 | |
mappu | bc4eb13058 | |
mappu | 1a7a435460 | |
mappu | c16736d86f | |
mappu | 9a833d025a | |
mappu | 525dcf31ad | |
mappu | ebb5f3cc7a | |
mappu | ad30f36365 | |
mappu | 020082d830 | |
mappu | 03b7fcdd89 |
|
@ -0,0 +1,2 @@
|
||||||
|
cmd/mkthumb/mkthumb
|
||||||
|
cmd/mkthumb/mkthumb.exe
|
5
.hgtags
5
.hgtags
|
@ -1,5 +0,0 @@
|
||||||
efd7b407177086c57e8c086605c2c8d1cee23840 release-1.0
|
|
||||||
292439a79182796c2f6277ef13ca179379b0fb88 v0.2.0
|
|
||||||
efd7b407177086c57e8c086605c2c8d1cee23840 v0.1.0
|
|
||||||
efd7b407177086c57e8c086605c2c8d1cee23840 release-1.0
|
|
||||||
0000000000000000000000000000000000000000 release-1.0
|
|
|
@ -0,0 +1,56 @@
|
||||||
|
package thumbnail
|
||||||
|
|
||||||
|
import (
|
||||||
|
"mime"
|
||||||
|
"path/filepath"
|
||||||
|
|
||||||
|
lru "github.com/hashicorp/golang-lru"
|
||||||
|
)
|
||||||
|
|
||||||
|
type CachingThumbnailer struct {
|
||||||
|
t Thumbnailer
|
||||||
|
cache *lru.Cache // threadsafe, might be nil
|
||||||
|
}
|
||||||
|
|
||||||
|
var _ Thumbnailer = &CachingThumbnailer{} // interface assertion
|
||||||
|
|
||||||
|
func NewCachingThumbnailer(cacheSize int, opts *Config) (Thumbnailer, error) {
|
||||||
|
|
||||||
|
upstream := NewThumbnailerEx(opts)
|
||||||
|
|
||||||
|
if cacheSize == 0 {
|
||||||
|
return upstream, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
thumbCache, err := lru.New(cacheSize)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return &CachingThumbnailer{
|
||||||
|
t: upstream,
|
||||||
|
cache: thumbCache,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (this *CachingThumbnailer) RenderFile(absPath string) ([]byte, error) {
|
||||||
|
mimeType := mime.TypeByExtension(filepath.Ext(absPath))
|
||||||
|
return this.RenderFileAs(absPath, mimeType)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (this *CachingThumbnailer) RenderFileAs(absPath, mimeType string) ([]byte, error) {
|
||||||
|
thumb, ok := this.cache.Get(absPath)
|
||||||
|
if ok {
|
||||||
|
return thumb.([]byte), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Add to cache
|
||||||
|
rendered, err := this.t.RenderFile(absPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
this.cache.Add(absPath, rendered)
|
||||||
|
|
||||||
|
return rendered, nil
|
||||||
|
}
|
|
@ -0,0 +1,67 @@
|
||||||
|
package thumbnail
|
||||||
|
|
||||||
|
import (
|
||||||
|
"image"
|
||||||
|
"mime"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
type DirectThumbnailer struct {
|
||||||
|
cfg Config
|
||||||
|
}
|
||||||
|
|
||||||
|
var _ Thumbnailer = &DirectThumbnailer{} // interface assertion
|
||||||
|
|
||||||
|
func NewThumbnailer(w, h int) Thumbnailer {
|
||||||
|
opts := DefaultConfig
|
||||||
|
opts.Width = w
|
||||||
|
opts.Height = h
|
||||||
|
|
||||||
|
return NewThumbnailerEx(&opts)
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewThumbnailerEx(opts *Config) Thumbnailer {
|
||||||
|
if opts == nil {
|
||||||
|
opts = &DefaultConfig
|
||||||
|
}
|
||||||
|
|
||||||
|
return &DirectThumbnailer{cfg: *opts}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (this *DirectThumbnailer) RenderFile(absPath string) ([]byte, error) {
|
||||||
|
mimeType := mime.TypeByExtension(filepath.Ext(absPath))
|
||||||
|
return this.RenderFileAs(absPath, mimeType)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (this *DirectThumbnailer) RenderFileAs(absPath, mimeType string) ([]byte, error) {
|
||||||
|
|
||||||
|
// Decode source
|
||||||
|
|
||||||
|
var src image.Image
|
||||||
|
var err error = nil
|
||||||
|
if strings.HasPrefix(mimeType, `image/`) {
|
||||||
|
src, err = this.imageSnapshot(absPath, mimeType)
|
||||||
|
|
||||||
|
} else if strings.HasPrefix(mimeType, `video/`) {
|
||||||
|
src, err = this.videoSnapshot(absPath)
|
||||||
|
|
||||||
|
} else {
|
||||||
|
return nil, ErrUnsupportedFiletype
|
||||||
|
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Scale
|
||||||
|
|
||||||
|
dest, err := this.scaleImage(src)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Rasterise result
|
||||||
|
|
||||||
|
return this.encode(dest)
|
||||||
|
}
|
|
@ -0,0 +1,48 @@
|
||||||
|
# thumbnail
|
||||||
|
|
||||||
|
A thumbnailing library for Go.
|
||||||
|
|
||||||
|
- Supports jpeg / png / gif / bmp / webp files (internally)
|
||||||
|
- Supports video files (requires `ffmpeg` in `$PATH`)
|
||||||
|
- Optional LRU cache of recent thumbnails for performance
|
||||||
|
- Sampling algorithms: Nearest-neighbour, Bilinear (fast), Bilinear (accurate), Bicubic (Catmull-Rom)
|
||||||
|
- Scaling algorithms: Fit inside, Fit outside, Stretch
|
||||||
|
- Output formats: JPG, PNG, Quantised/lossy PNG (via `go-imagequant` if CGO is available), BMP
|
||||||
|
|
||||||
|
A standalone binary `mkthumb` is provided as a sample utility.
|
||||||
|
|
||||||
|
This package can be installed via go get: `go get code.ivysaur.me/thumbnail`
|
||||||
|
|
||||||
|
## Changelog
|
||||||
|
|
||||||
|
2020-07-25 1.0.2
|
||||||
|
- Automatically build pngquant support if CGO is available
|
||||||
|
- Remove vendor directory, rely more heavily on Go Modules
|
||||||
|
- Update module dependencies
|
||||||
|
|
||||||
|
2018-12-31 1.0.1
|
||||||
|
- Convert to Go Modules
|
||||||
|
- Update vendored dependencies
|
||||||
|
|
||||||
|
2018-06-09 1.0.0
|
||||||
|
- Feature: Major speed improvement
|
||||||
|
- Feature: Support FitInside, FitOutside, and Stretch stretching modes
|
||||||
|
- Feature: Support Bilnear (Fast), Bilnear (Accurate), and Bicubic (Catmull-Rom) scaling algorithms
|
||||||
|
- Feature: Support PNG output without imagequant; support BMP output
|
||||||
|
- Feature: Support Webp and BMP input files
|
||||||
|
- Fix wrong dimension output for video input files
|
||||||
|
- Fix jagged output of bilinear resizer
|
||||||
|
- [⬇️ thumbnail-1.0.0-src.tar.xz](https://git.ivysaur.me/attachments/aa9c5b40-3faa-45c3-a45e-10004d03481c) *(4.78 KiB)*
|
||||||
|
|
||||||
|
2018-06-04 0.2.1
|
||||||
|
- Add `disableimagecrush` build tag option to exclude cgo library dependency
|
||||||
|
- [⬇️ thumbnail-0.2.1-src.zip](https://git.ivysaur.me/attachments/08c12811-fc7b-44b0-93fd-161091632e9e) *(24.09 KiB)*
|
||||||
|
|
||||||
|
2017-11-18 0.2.0
|
||||||
|
- Initial standalone release
|
||||||
|
- Feature: Decode input with specified mime type
|
||||||
|
- Feature: Allow passing zero as thumbnail cache size
|
||||||
|
- [⬇️ thumbnail-0.2.0-src.zip](https://git.ivysaur.me/attachments/c2405a1a-50c3-4059-9b49-a77bc2ee311f) *(23.37 KiB)*
|
||||||
|
|
||||||
|
2017-01-03 0.1.0
|
||||||
|
- Version of `thumbnail` vendored with `webdir` 1.0 (previously tagged as `release-1.0`)
|
145
Thumbnailer.go
145
Thumbnailer.go
|
@ -2,34 +2,7 @@ package thumbnail
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"errors"
|
"errors"
|
||||||
"image/gif"
|
|
||||||
"image/jpeg"
|
|
||||||
"image/png"
|
|
||||||
"mime"
|
|
||||||
"os"
|
|
||||||
"path/filepath"
|
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
lru "github.com/hashicorp/golang-lru"
|
|
||||||
)
|
|
||||||
|
|
||||||
type OutputFormat uint8
|
|
||||||
type AspectFormat uint8
|
|
||||||
type ScaleFormat uint8
|
|
||||||
|
|
||||||
const (
|
|
||||||
OUTPUT_PNG_CRUSH OutputFormat = 2
|
|
||||||
OUTPUT_JPG OutputFormat = 3
|
|
||||||
OUTPUT__DEFAULT = OUTPUT_PNG_CRUSH
|
|
||||||
|
|
||||||
ASPECT_PAD_TO_DIMENSIONS AspectFormat = 80
|
|
||||||
ASPECT_RESPECT_MAX_DIMENSION_ONLY AspectFormat = 81
|
|
||||||
ASPECT_CROP_TO_DIMENSIONS AspectFormat = 82
|
|
||||||
ASPECT__DEFAULT = ASPECT_PAD_TO_DIMENSIONS
|
|
||||||
|
|
||||||
SCALEFMT_NN ScaleFormat = 120
|
|
||||||
SCALEFMT_BILINEAR ScaleFormat = 121
|
|
||||||
SCALEFMT__DEFAULT = SCALEFMT_BILINEAR
|
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
|
@ -37,123 +10,19 @@ var (
|
||||||
ErrUnsupportedFiletype error = errors.New("Unsupported filetype")
|
ErrUnsupportedFiletype error = errors.New("Unsupported filetype")
|
||||||
)
|
)
|
||||||
|
|
||||||
type Thumbnailer struct {
|
type Thumbnailer interface {
|
||||||
width int
|
RenderFile(absPath string) ([]byte, error)
|
||||||
height int
|
RenderFileAs(absPath, mimeType string) ([]byte, error)
|
||||||
ofmt OutputFormat
|
|
||||||
afmt AspectFormat
|
|
||||||
sfmt ScaleFormat
|
|
||||||
|
|
||||||
thumbCache *lru.Cache // threadsafe, might be nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewThumbnailer(Width, Height int, MaxCacheSize uint) *Thumbnailer {
|
|
||||||
return NewThumbnailerEx(Width, Height, MaxCacheSize, OUTPUT__DEFAULT, ASPECT__DEFAULT, SCALEFMT__DEFAULT)
|
|
||||||
}
|
|
||||||
|
|
||||||
func NewThumbnailerEx(Width, Height int, MaxCacheSize uint, of OutputFormat, af AspectFormat, sf ScaleFormat) *Thumbnailer {
|
|
||||||
|
|
||||||
ret := &Thumbnailer{
|
|
||||||
width: Width,
|
|
||||||
height: Height,
|
|
||||||
ofmt: of,
|
|
||||||
afmt: af,
|
|
||||||
sfmt: sf,
|
|
||||||
thumbCache: nil,
|
|
||||||
}
|
|
||||||
|
|
||||||
if MaxCacheSize > 0 {
|
|
||||||
thumbCache, err := lru.New(int(MaxCacheSize))
|
|
||||||
if err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
ret.thumbCache = thumbCache
|
|
||||||
}
|
|
||||||
|
|
||||||
return ret
|
|
||||||
}
|
|
||||||
|
|
||||||
func (this *Thumbnailer) Width() int {
|
|
||||||
return this.width
|
|
||||||
}
|
|
||||||
|
|
||||||
func (this *Thumbnailer) Height() int {
|
|
||||||
return this.height
|
|
||||||
}
|
|
||||||
|
|
||||||
func (this *Thumbnailer) RenderFile(absPath string) ([]byte, error) {
|
|
||||||
|
|
||||||
if this.thumbCache != nil {
|
|
||||||
thumb, ok := this.thumbCache.Get(absPath)
|
|
||||||
if ok {
|
|
||||||
return thumb.([]byte), nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add to cache
|
|
||||||
thumb, err := this.RenderFile_NoCache(absPath)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
if this.thumbCache != nil {
|
|
||||||
this.thumbCache.Add(absPath, thumb)
|
|
||||||
}
|
|
||||||
|
|
||||||
return thumb, nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func FiletypeSupported(ext string) bool {
|
func FiletypeSupported(ext string) bool {
|
||||||
switch strings.ToLower(ext) {
|
switch strings.ToLower(ext) {
|
||||||
case ".jpg", ".jpeg", ".png", ".gif", ".avi", ".mkv", ".mp4", ".ogm", ".wmv", ".flv", ".rm", ".rmvb":
|
case
|
||||||
|
".jpg", ".jpeg", ".png", ".gif",
|
||||||
|
".avi", ".mkv", ".mp4", ".ogm", ".wmv", ".flv", ".rm", ".rmvb",
|
||||||
|
".bmp", ".webp":
|
||||||
return true
|
return true
|
||||||
default:
|
default:
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (this *Thumbnailer) RenderFile_NoCache(absPath string) ([]byte, error) {
|
|
||||||
return this.RenderFile_NoCache_MimeType(absPath, mime.TypeByExtension(filepath.Ext(absPath)))
|
|
||||||
}
|
|
||||||
|
|
||||||
func (this *Thumbnailer) RenderFile_NoCache_MimeType(absPath, mimeType string) ([]byte, error) {
|
|
||||||
|
|
||||||
fh, err := os.OpenFile(absPath, os.O_RDONLY, 0400)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
defer fh.Close()
|
|
||||||
|
|
||||||
if mimeType == `image/jpeg` {
|
|
||||||
src, err := jpeg.Decode(fh)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return this.RenderScaledImage(src)
|
|
||||||
|
|
||||||
} else if mimeType == `image/png` {
|
|
||||||
src, err := png.Decode(fh)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return this.RenderScaledImage(src)
|
|
||||||
|
|
||||||
} else if mimeType == `image/gif` {
|
|
||||||
src, err := gif.Decode(fh)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return this.RenderScaledImage(src)
|
|
||||||
|
|
||||||
} else if strings.HasPrefix(mimeType, `video/`) {
|
|
||||||
return this.RenderScaledFfmpeg(absPath)
|
|
||||||
|
|
||||||
} else {
|
|
||||||
return nil, ErrUnsupportedFiletype
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
|
@ -1,27 +0,0 @@
|
||||||
A thumbnailing library for Go.
|
|
||||||
|
|
||||||
Written in Go
|
|
||||||
|
|
||||||
Earlier versions of this project were vendored into `webdir`.
|
|
||||||
|
|
||||||
- Supports jpeg / png / gif files (internally) and video files (requires `ffmpeg` in `$PATH`)
|
|
||||||
- LRU cache of recent thumbnails for performance
|
|
||||||
- Scaling algorithms: Nearest-neighbour or bilinear
|
|
||||||
- Aspect ratio preserving (fit outside with image crop; fit inside with transparent background; fit inside with dimension reduction)
|
|
||||||
- Output formats: JPG (internal) or transparent PNG (via `go-imagequant`)
|
|
||||||
|
|
||||||
A standalone binary `mkthumb` is provided as a sample utility.
|
|
||||||
|
|
||||||
=CHANGELOG=
|
|
||||||
|
|
||||||
2018-05-04 0.2.1
|
|
||||||
- Add `disableimagecrush` build tag option to exclude cgo library dependency
|
|
||||||
|
|
||||||
2017-11-18 0.2.0
|
|
||||||
- Initial standalone release
|
|
||||||
- Feature: Decode input with specified mime type
|
|
||||||
- Feature: Allow passing zero as thumbnail cache size
|
|
||||||
|
|
||||||
2017-01-03 0.1.0
|
|
||||||
- Version of `thumbnail` vendored with `webdir` 1.0 (previously tagged as `release-1.0`)
|
|
||||||
|
|
|
@ -1,35 +0,0 @@
|
||||||
package thumbnail
|
|
||||||
|
|
||||||
import (
|
|
||||||
"image/color"
|
|
||||||
)
|
|
||||||
|
|
||||||
func Blend(a, b color.Color) color.Color {
|
|
||||||
switch a.(type) {
|
|
||||||
case color.RGBA:
|
|
||||||
return BlendRGBA(a.(color.RGBA), b.(color.RGBA)) // FIXME there's syntax for this
|
|
||||||
|
|
||||||
case color.YCbCr:
|
|
||||||
return BlendYCbCr(a.(color.YCbCr), b.(color.YCbCr))
|
|
||||||
|
|
||||||
default:
|
|
||||||
return a // ??? unknown color format
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func BlendYCbCr(a, b color.YCbCr) color.YCbCr {
|
|
||||||
return color.YCbCr{
|
|
||||||
Y: uint8((int(a.Y) + int(b.Y)) / 2),
|
|
||||||
Cb: uint8((int(a.Cb) + int(b.Cb)) / 2),
|
|
||||||
Cr: uint8((int(a.Cr) + int(b.Cr)) / 2),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func BlendRGBA(a, b color.RGBA) color.RGBA {
|
|
||||||
return color.RGBA{
|
|
||||||
R: uint8((int(a.R) + int(b.R)) / 2),
|
|
||||||
G: uint8((int(a.G) + int(b.G)) / 2),
|
|
||||||
B: uint8((int(a.B) + int(b.B)) / 2),
|
|
||||||
A: uint8((int(a.A) + int(b.A)) / 2),
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,9 +0,0 @@
|
||||||
.PHONY: all
|
|
||||||
|
|
||||||
all: mkthumb
|
|
||||||
|
|
||||||
clean:
|
|
||||||
rm ./mkthumb
|
|
||||||
|
|
||||||
mkthumb: mkthumb.go
|
|
||||||
go build -ldflags '-s -w'
|
|
|
@ -20,8 +20,8 @@ func main() {
|
||||||
|
|
||||||
fname := flag.Arg(0)
|
fname := flag.Arg(0)
|
||||||
|
|
||||||
th := thumbnail.NewThumbnailer(*width, *height, 1)
|
th := thumbnail.NewThumbnailer(*width, *height)
|
||||||
data, err := th.RenderFile_NoCache(fname)
|
data, err := th.RenderFile(fname)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
fmt.Fprintln(os.Stderr, err.Error())
|
fmt.Fprintln(os.Stderr, err.Error())
|
||||||
os.Exit(1)
|
os.Exit(1)
|
|
@ -0,0 +1,37 @@
|
||||||
|
package thumbnail
|
||||||
|
|
||||||
|
type OutputFormat uint8
|
||||||
|
type AspectFormat uint8
|
||||||
|
type ScaleFormat uint8
|
||||||
|
|
||||||
|
const (
|
||||||
|
Png OutputFormat = 1
|
||||||
|
PngCrush OutputFormat = 2
|
||||||
|
Jpeg OutputFormat = 3
|
||||||
|
Bmp OutputFormat = 4
|
||||||
|
|
||||||
|
FitOutside AspectFormat = 80 // Pad out with black bars to dimensions
|
||||||
|
FitInside AspectFormat = 82 // Crop to dimensions
|
||||||
|
Stretch AspectFormat = 83 // Doesn't preserve aspect ratio
|
||||||
|
|
||||||
|
NearestNeighbour ScaleFormat = 120
|
||||||
|
BilinearFast ScaleFormat = 121
|
||||||
|
BilinearAccurate ScaleFormat = 122
|
||||||
|
Bicubic ScaleFormat = 123
|
||||||
|
)
|
||||||
|
|
||||||
|
type Config struct {
|
||||||
|
Width int
|
||||||
|
Height int
|
||||||
|
Output OutputFormat
|
||||||
|
Aspect AspectFormat
|
||||||
|
Scale ScaleFormat
|
||||||
|
}
|
||||||
|
|
||||||
|
var DefaultConfig = Config{
|
||||||
|
Width: 128,
|
||||||
|
Height: 128,
|
||||||
|
Output: Jpeg,
|
||||||
|
Aspect: FitOutside,
|
||||||
|
Scale: Bicubic,
|
||||||
|
}
|
|
@ -0,0 +1,43 @@
|
||||||
|
package thumbnail
|
||||||
|
|
||||||
|
import (
|
||||||
|
"image"
|
||||||
|
"image/gif"
|
||||||
|
"image/jpeg"
|
||||||
|
"image/png"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"golang.org/x/image/bmp"
|
||||||
|
"golang.org/x/image/webp"
|
||||||
|
)
|
||||||
|
|
||||||
|
type imageDecoder func(io.Reader) (image.Image, error)
|
||||||
|
|
||||||
|
var imageDecoders map[string]imageDecoder = map[string]imageDecoder{
|
||||||
|
`image/jpeg`: jpeg.Decode,
|
||||||
|
`image/png`: png.Decode,
|
||||||
|
`image/gif`: gif.Decode,
|
||||||
|
`image/webp`: webp.Decode,
|
||||||
|
`image/bmp`: bmp.Decode,
|
||||||
|
}
|
||||||
|
|
||||||
|
func (this *DirectThumbnailer) imageSnapshot(absPath, mimeType string) (image.Image, error) {
|
||||||
|
|
||||||
|
// Load file from path
|
||||||
|
|
||||||
|
fh, err := os.OpenFile(absPath, os.O_RDONLY, 0400)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer fh.Close()
|
||||||
|
|
||||||
|
// Decode image
|
||||||
|
|
||||||
|
fn, ok := imageDecoders[mimeType]
|
||||||
|
if !ok {
|
||||||
|
return nil, ErrUnsupportedFiletype
|
||||||
|
}
|
||||||
|
|
||||||
|
return fn(fh)
|
||||||
|
}
|
|
@ -0,0 +1,52 @@
|
||||||
|
package thumbnail
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"image"
|
||||||
|
"image/png"
|
||||||
|
"io"
|
||||||
|
"os/exec"
|
||||||
|
)
|
||||||
|
|
||||||
|
func (this *DirectThumbnailer) videoSnapshot(absPath string) (image.Image, error) {
|
||||||
|
|
||||||
|
cmd := exec.Command(
|
||||||
|
"ffmpeg",
|
||||||
|
"-loglevel", "0",
|
||||||
|
"-timelimit", "10", // seconds
|
||||||
|
"-an",
|
||||||
|
"-i", absPath,
|
||||||
|
"-vf", `thumbnail`,
|
||||||
|
"-frames:v", "1",
|
||||||
|
"-f", "image2pipe",
|
||||||
|
"-c:v", "png", // always PNG output - we will resample/rescale it ourselves
|
||||||
|
"-",
|
||||||
|
)
|
||||||
|
|
||||||
|
// -ss 00:00:30
|
||||||
|
|
||||||
|
stdout, err := cmd.StdoutPipe()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
err = cmd.Start()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
out := bytes.Buffer{}
|
||||||
|
_, err = io.Copy(&out, stdout)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
err = cmd.Wait()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// Try to decode as PNG image
|
||||||
|
|
||||||
|
return png.Decode(bytes.NewReader(out.Bytes()))
|
||||||
|
}
|
3
doc.go
3
doc.go
|
@ -1,3 +0,0 @@
|
||||||
// Package thumbnail contains functions for taking the thumbnail of image and
|
|
||||||
// video files, and caching the result for performance.
|
|
||||||
package thumbnail
|
|
|
@ -0,0 +1,9 @@
|
||||||
|
module code.ivysaur.me/thumbnail
|
||||||
|
|
||||||
|
require (
|
||||||
|
code.ivysaur.me/imagequant/v2 v2.12.6
|
||||||
|
github.com/hashicorp/golang-lru v0.5.4
|
||||||
|
golang.org/x/image v0.0.0-20200618115811-c13761719519
|
||||||
|
)
|
||||||
|
|
||||||
|
go 1.13
|
|
@ -0,0 +1,13 @@
|
||||||
|
code.ivysaur.me/imagequant/v2 v2.12.6 h1:xYrGj6GOdAcutmzqBxG7bDZ70r4jYHADOCZ+ktyMU3Y=
|
||||||
|
code.ivysaur.me/imagequant/v2 v2.12.6/go.mod h1:seCAm0sP2IBsb1YNBj4D+EZovIuGe16+6Xo0aiGyhDU=
|
||||||
|
github.com/hashicorp/golang-lru v0.0.0-20180201235237-0fb14efe8c47 h1:UnszMmmmm5vLwWzDjTFVIkfhvWF1NdrmChl8L2NUDCw=
|
||||||
|
github.com/hashicorp/golang-lru v0.0.0-20180201235237-0fb14efe8c47/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||||
|
github.com/hashicorp/golang-lru v0.5.0 h1:CL2msUPvZTLb5O648aiLNJw3hnBxN2+1Jq8rCOH9wdo=
|
||||||
|
github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
|
||||||
|
github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
|
||||||
|
github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
|
||||||
|
golang.org/x/image v0.0.0-20180601115456-af66defab954 h1:n7UB+yxe5jyWxOA3BTAfwi23lhfKEIddaB/so7YOYe0=
|
||||||
|
golang.org/x/image v0.0.0-20180601115456-af66defab954/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
|
||||||
|
golang.org/x/image v0.0.0-20200618115811-c13761719519 h1:1e2ufUJNM3lCHEY5jIgac/7UTjd6cgJNdatjPdFWf34=
|
||||||
|
golang.org/x/image v0.0.0-20200618115811-c13761719519/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
|
||||||
|
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
77
image.go
77
image.go
|
@ -1,77 +0,0 @@
|
||||||
package thumbnail
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"image"
|
|
||||||
"image/jpeg"
|
|
||||||
)
|
|
||||||
|
|
||||||
func (this *Thumbnailer) RenderScaledImage(src image.Image) ([]byte, error) {
|
|
||||||
srcW := src.Bounds().Max.X
|
|
||||||
srcH := src.Bounds().Max.Y
|
|
||||||
destW := 0
|
|
||||||
destH := 0
|
|
||||||
|
|
||||||
if srcW > srcH {
|
|
||||||
destW = this.width
|
|
||||||
destH = this.height * srcH / srcW
|
|
||||||
} else {
|
|
||||||
destW = this.width * srcW / srcH
|
|
||||||
destH = this.height
|
|
||||||
}
|
|
||||||
|
|
||||||
offsetX := (this.width - destW) / 2
|
|
||||||
offsetY := (this.height - destH) / 2
|
|
||||||
|
|
||||||
scaleW := float64(srcW) / float64(destW)
|
|
||||||
scaleH := float64(srcH) / float64(destH)
|
|
||||||
|
|
||||||
dest := image.NewRGBA(image.Rectangle{Max: image.Point{X: this.width, Y: this.height}})
|
|
||||||
|
|
||||||
switch this.sfmt {
|
|
||||||
|
|
||||||
case SCALEFMT_BILINEAR:
|
|
||||||
|
|
||||||
for y := 0; y < destH; y += 1 {
|
|
||||||
for x := 0; x < destW; x += 1 {
|
|
||||||
c00 := src.At(int(float64(x)*scaleW), int(float64(y)*scaleH))
|
|
||||||
c01 := src.At(int((float64(x)+0.5)*scaleW), int(float64(y)*scaleH))
|
|
||||||
c10 := src.At(int(float64(x)*scaleW), int((float64(y)+0.5)*scaleH))
|
|
||||||
c11 := src.At(int((float64(x)+0.5)*scaleW), int((float64(y)+0.5)*scaleH))
|
|
||||||
cBlend := Blend(Blend(c00, c01), Blend(c10, c11))
|
|
||||||
dest.Set(x+offsetX, y+offsetY, cBlend)
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
case SCALEFMT_NN:
|
|
||||||
|
|
||||||
for y := 0; y < destH; y += 1 {
|
|
||||||
for x := 0; x < destW; x += 1 {
|
|
||||||
mapx := int(float64(x) * scaleW)
|
|
||||||
mapy := int(float64(y) * scaleH)
|
|
||||||
dest.Set(x+offsetX, y+offsetY, src.At(mapx, mapy))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
switch this.ofmt {
|
|
||||||
case OUTPUT_PNG_CRUSH:
|
|
||||||
return crushFast(dest)
|
|
||||||
|
|
||||||
case OUTPUT_JPG:
|
|
||||||
buff := bytes.Buffer{}
|
|
||||||
err := jpeg.Encode(&buff, dest, &jpeg.Options{Quality: jpeg.DefaultQuality})
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return buff.Bytes(), nil
|
|
||||||
|
|
||||||
default:
|
|
||||||
return nil, ErrInvalidOption
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
|
@ -1,4 +1,4 @@
|
||||||
//+build !disableimagecrush
|
//+build cgo
|
||||||
|
|
||||||
package thumbnail
|
package thumbnail
|
||||||
|
|
||||||
|
@ -9,7 +9,7 @@ import (
|
||||||
"image/color"
|
"image/color"
|
||||||
"image/png"
|
"image/png"
|
||||||
|
|
||||||
"code.ivysaur.me/imagequant"
|
"code.ivysaur.me/imagequant/v2"
|
||||||
)
|
)
|
||||||
|
|
||||||
func goImageToRgba32(im image.Image) []byte {
|
func goImageToRgba32(im image.Image) []byte {
|
|
@ -1,4 +1,4 @@
|
||||||
// +build disableimagecrush
|
//+build !cgo
|
||||||
|
|
||||||
package thumbnail
|
package thumbnail
|
||||||
|
|
|
@ -1 +1 @@
|
||||||
hg archive out.zip
|
XZ_OPTS=-9 tar caf out.tar.xz --owner=0 --group=0 *.go Gopkg* cmd/mkthumb/*.go _dist/README.txt
|
||||||
|
|
|
@ -0,0 +1,122 @@
|
||||||
|
package thumbnail
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"image"
|
||||||
|
"image/jpeg"
|
||||||
|
"image/png"
|
||||||
|
|
||||||
|
"golang.org/x/image/bmp"
|
||||||
|
"golang.org/x/image/draw"
|
||||||
|
)
|
||||||
|
|
||||||
|
func (this *DirectThumbnailer) scaleImage(src image.Image) (image.Image, error) {
|
||||||
|
srcW := src.Bounds().Max.X
|
||||||
|
srcH := src.Bounds().Max.Y
|
||||||
|
var srcCopyPosition, destCopyPosition image.Rectangle
|
||||||
|
|
||||||
|
switch this.cfg.Aspect {
|
||||||
|
case FitInside:
|
||||||
|
|
||||||
|
var destW, destH int
|
||||||
|
|
||||||
|
if srcW > srcH {
|
||||||
|
destW = this.cfg.Width
|
||||||
|
destH = this.cfg.Height * srcH / srcW
|
||||||
|
} else {
|
||||||
|
destW = this.cfg.Width * srcW / srcH
|
||||||
|
destH = this.cfg.Height
|
||||||
|
}
|
||||||
|
|
||||||
|
offsetX := (this.cfg.Width - destW) / 2
|
||||||
|
offsetY := (this.cfg.Height - destH) / 2
|
||||||
|
|
||||||
|
srcCopyPosition = src.Bounds()
|
||||||
|
destCopyPosition = image.Rect(offsetX, offsetY, destW+offsetX, destH+offsetY)
|
||||||
|
|
||||||
|
case FitOutside:
|
||||||
|
|
||||||
|
var srcSmallestDim, offsetX, offsetY int
|
||||||
|
if srcW > srcH {
|
||||||
|
// Landscape
|
||||||
|
srcSmallestDim = srcH
|
||||||
|
offsetY = 0
|
||||||
|
offsetX = (srcW - srcH) / 2
|
||||||
|
} else {
|
||||||
|
// Portrait (or square)
|
||||||
|
srcSmallestDim = srcW
|
||||||
|
offsetY = (srcH - srcW) / 2
|
||||||
|
offsetX = 0
|
||||||
|
}
|
||||||
|
|
||||||
|
srcCopyPosition = image.Rect(offsetX, offsetY, srcSmallestDim+offsetX, srcSmallestDim+offsetY)
|
||||||
|
destCopyPosition = image.Rect(0, 0, this.cfg.Width, this.cfg.Height)
|
||||||
|
|
||||||
|
case Stretch:
|
||||||
|
srcCopyPosition = src.Bounds()
|
||||||
|
destCopyPosition = image.Rect(0, 0, this.cfg.Width, this.cfg.Height)
|
||||||
|
|
||||||
|
default:
|
||||||
|
return nil, ErrInvalidOption
|
||||||
|
}
|
||||||
|
|
||||||
|
//
|
||||||
|
|
||||||
|
dest := image.NewRGBA(image.Rect(0, 0, this.cfg.Width, this.cfg.Height))
|
||||||
|
|
||||||
|
// For a transparent destination, Op.Src is faster than Op.Over
|
||||||
|
|
||||||
|
switch this.cfg.Scale {
|
||||||
|
case NearestNeighbour:
|
||||||
|
draw.NearestNeighbor.Scale(dest, destCopyPosition, src, srcCopyPosition, draw.Src, nil)
|
||||||
|
|
||||||
|
case BilinearFast:
|
||||||
|
draw.ApproxBiLinear.Scale(dest, destCopyPosition, src, srcCopyPosition, draw.Src, nil)
|
||||||
|
|
||||||
|
case BilinearAccurate:
|
||||||
|
draw.BiLinear.Scale(dest, destCopyPosition, src, srcCopyPosition, draw.Src, nil)
|
||||||
|
|
||||||
|
case Bicubic:
|
||||||
|
draw.CatmullRom.Scale(dest, destCopyPosition, src, srcCopyPosition, draw.Src, nil)
|
||||||
|
}
|
||||||
|
|
||||||
|
return dest, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (this *DirectThumbnailer) encode(dest image.Image) ([]byte, error) {
|
||||||
|
|
||||||
|
switch this.cfg.Output {
|
||||||
|
case Png:
|
||||||
|
buff := bytes.Buffer{}
|
||||||
|
err := png.Encode(&buff, dest)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return buff.Bytes(), nil
|
||||||
|
|
||||||
|
case PngCrush:
|
||||||
|
return crushFast(dest)
|
||||||
|
|
||||||
|
case Jpeg:
|
||||||
|
buff := bytes.Buffer{}
|
||||||
|
err := jpeg.Encode(&buff, dest, &jpeg.Options{Quality: jpeg.DefaultQuality})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return buff.Bytes(), nil
|
||||||
|
|
||||||
|
case Bmp:
|
||||||
|
buff := bytes.Buffer{}
|
||||||
|
err := bmp.Encode(&buff, dest)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return buff.Bytes(), nil
|
||||||
|
|
||||||
|
default:
|
||||||
|
return nil, ErrInvalidOption
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
|
@ -1,23 +0,0 @@
|
||||||
# Compiled Object files, Static and Dynamic libs (Shared Objects)
|
|
||||||
*.o
|
|
||||||
*.a
|
|
||||||
*.so
|
|
||||||
|
|
||||||
# Folders
|
|
||||||
_obj
|
|
||||||
_test
|
|
||||||
|
|
||||||
# Architecture specific extensions/prefixes
|
|
||||||
*.[568vq]
|
|
||||||
[568vq].out
|
|
||||||
|
|
||||||
*.cgo1.go
|
|
||||||
*.cgo2.c
|
|
||||||
_cgo_defun.c
|
|
||||||
_cgo_gotypes.go
|
|
||||||
_cgo_export.*
|
|
||||||
|
|
||||||
_testmain.go
|
|
||||||
|
|
||||||
*.exe
|
|
||||||
*.test
|
|
|
@ -1,212 +0,0 @@
|
||||||
package lru
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
"sync"
|
|
||||||
|
|
||||||
"github.com/hashicorp/golang-lru/simplelru"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
// Default2QRecentRatio is the ratio of the 2Q cache dedicated
|
|
||||||
// to recently added entries that have only been accessed once.
|
|
||||||
Default2QRecentRatio = 0.25
|
|
||||||
|
|
||||||
// Default2QGhostEntries is the default ratio of ghost
|
|
||||||
// entries kept to track entries recently evicted
|
|
||||||
Default2QGhostEntries = 0.50
|
|
||||||
)
|
|
||||||
|
|
||||||
// TwoQueueCache is a thread-safe fixed size 2Q cache.
|
|
||||||
// 2Q is an enhancement over the standard LRU cache
|
|
||||||
// in that it tracks both frequently and recently used
|
|
||||||
// entries separately. This avoids a burst in access to new
|
|
||||||
// entries from evicting frequently used entries. It adds some
|
|
||||||
// additional tracking overhead to the standard LRU cache, and is
|
|
||||||
// computationally about 2x the cost, and adds some metadata over
|
|
||||||
// head. The ARCCache is similar, but does not require setting any
|
|
||||||
// parameters.
|
|
||||||
type TwoQueueCache struct {
|
|
||||||
size int
|
|
||||||
recentSize int
|
|
||||||
|
|
||||||
recent *simplelru.LRU
|
|
||||||
frequent *simplelru.LRU
|
|
||||||
recentEvict *simplelru.LRU
|
|
||||||
lock sync.RWMutex
|
|
||||||
}
|
|
||||||
|
|
||||||
// New2Q creates a new TwoQueueCache using the default
|
|
||||||
// values for the parameters.
|
|
||||||
func New2Q(size int) (*TwoQueueCache, error) {
|
|
||||||
return New2QParams(size, Default2QRecentRatio, Default2QGhostEntries)
|
|
||||||
}
|
|
||||||
|
|
||||||
// New2QParams creates a new TwoQueueCache using the provided
|
|
||||||
// parameter values.
|
|
||||||
func New2QParams(size int, recentRatio float64, ghostRatio float64) (*TwoQueueCache, error) {
|
|
||||||
if size <= 0 {
|
|
||||||
return nil, fmt.Errorf("invalid size")
|
|
||||||
}
|
|
||||||
if recentRatio < 0.0 || recentRatio > 1.0 {
|
|
||||||
return nil, fmt.Errorf("invalid recent ratio")
|
|
||||||
}
|
|
||||||
if ghostRatio < 0.0 || ghostRatio > 1.0 {
|
|
||||||
return nil, fmt.Errorf("invalid ghost ratio")
|
|
||||||
}
|
|
||||||
|
|
||||||
// Determine the sub-sizes
|
|
||||||
recentSize := int(float64(size) * recentRatio)
|
|
||||||
evictSize := int(float64(size) * ghostRatio)
|
|
||||||
|
|
||||||
// Allocate the LRUs
|
|
||||||
recent, err := simplelru.NewLRU(size, nil)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
frequent, err := simplelru.NewLRU(size, nil)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
recentEvict, err := simplelru.NewLRU(evictSize, nil)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// Initialize the cache
|
|
||||||
c := &TwoQueueCache{
|
|
||||||
size: size,
|
|
||||||
recentSize: recentSize,
|
|
||||||
recent: recent,
|
|
||||||
frequent: frequent,
|
|
||||||
recentEvict: recentEvict,
|
|
||||||
}
|
|
||||||
return c, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *TwoQueueCache) Get(key interface{}) (interface{}, bool) {
|
|
||||||
c.lock.Lock()
|
|
||||||
defer c.lock.Unlock()
|
|
||||||
|
|
||||||
// Check if this is a frequent value
|
|
||||||
if val, ok := c.frequent.Get(key); ok {
|
|
||||||
return val, ok
|
|
||||||
}
|
|
||||||
|
|
||||||
// If the value is contained in recent, then we
|
|
||||||
// promote it to frequent
|
|
||||||
if val, ok := c.recent.Peek(key); ok {
|
|
||||||
c.recent.Remove(key)
|
|
||||||
c.frequent.Add(key, val)
|
|
||||||
return val, ok
|
|
||||||
}
|
|
||||||
|
|
||||||
// No hit
|
|
||||||
return nil, false
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *TwoQueueCache) Add(key, value interface{}) {
|
|
||||||
c.lock.Lock()
|
|
||||||
defer c.lock.Unlock()
|
|
||||||
|
|
||||||
// Check if the value is frequently used already,
|
|
||||||
// and just update the value
|
|
||||||
if c.frequent.Contains(key) {
|
|
||||||
c.frequent.Add(key, value)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check if the value is recently used, and promote
|
|
||||||
// the value into the frequent list
|
|
||||||
if c.recent.Contains(key) {
|
|
||||||
c.recent.Remove(key)
|
|
||||||
c.frequent.Add(key, value)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// If the value was recently evicted, add it to the
|
|
||||||
// frequently used list
|
|
||||||
if c.recentEvict.Contains(key) {
|
|
||||||
c.ensureSpace(true)
|
|
||||||
c.recentEvict.Remove(key)
|
|
||||||
c.frequent.Add(key, value)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add to the recently seen list
|
|
||||||
c.ensureSpace(false)
|
|
||||||
c.recent.Add(key, value)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// ensureSpace is used to ensure we have space in the cache
|
|
||||||
func (c *TwoQueueCache) ensureSpace(recentEvict bool) {
|
|
||||||
// If we have space, nothing to do
|
|
||||||
recentLen := c.recent.Len()
|
|
||||||
freqLen := c.frequent.Len()
|
|
||||||
if recentLen+freqLen < c.size {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// If the recent buffer is larger than
|
|
||||||
// the target, evict from there
|
|
||||||
if recentLen > 0 && (recentLen > c.recentSize || (recentLen == c.recentSize && !recentEvict)) {
|
|
||||||
k, _, _ := c.recent.RemoveOldest()
|
|
||||||
c.recentEvict.Add(k, nil)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Remove from the frequent list otherwise
|
|
||||||
c.frequent.RemoveOldest()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *TwoQueueCache) Len() int {
|
|
||||||
c.lock.RLock()
|
|
||||||
defer c.lock.RUnlock()
|
|
||||||
return c.recent.Len() + c.frequent.Len()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *TwoQueueCache) Keys() []interface{} {
|
|
||||||
c.lock.RLock()
|
|
||||||
defer c.lock.RUnlock()
|
|
||||||
k1 := c.frequent.Keys()
|
|
||||||
k2 := c.recent.Keys()
|
|
||||||
return append(k1, k2...)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *TwoQueueCache) Remove(key interface{}) {
|
|
||||||
c.lock.Lock()
|
|
||||||
defer c.lock.Unlock()
|
|
||||||
if c.frequent.Remove(key) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if c.recent.Remove(key) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if c.recentEvict.Remove(key) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *TwoQueueCache) Purge() {
|
|
||||||
c.lock.Lock()
|
|
||||||
defer c.lock.Unlock()
|
|
||||||
c.recent.Purge()
|
|
||||||
c.frequent.Purge()
|
|
||||||
c.recentEvict.Purge()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *TwoQueueCache) Contains(key interface{}) bool {
|
|
||||||
c.lock.RLock()
|
|
||||||
defer c.lock.RUnlock()
|
|
||||||
return c.frequent.Contains(key) || c.recent.Contains(key)
|
|
||||||
}
|
|
||||||
|
|
||||||
func (c *TwoQueueCache) Peek(key interface{}) (interface{}, bool) {
|
|
||||||
c.lock.RLock()
|
|
||||||
defer c.lock.RUnlock()
|
|
||||||
if val, ok := c.frequent.Peek(key); ok {
|
|
||||||
return val, ok
|
|
||||||
}
|
|
||||||
return c.recent.Peek(key)
|
|
||||||
}
|
|
|
@ -1,306 +0,0 @@
|
||||||
package lru
|
|
||||||
|
|
||||||
import (
|
|
||||||
"math/rand"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func Benchmark2Q_Rand(b *testing.B) {
|
|
||||||
l, err := New2Q(8192)
|
|
||||||
if err != nil {
|
|
||||||
b.Fatalf("err: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
trace := make([]int64, b.N*2)
|
|
||||||
for i := 0; i < b.N*2; i++ {
|
|
||||||
trace[i] = rand.Int63() % 32768
|
|
||||||
}
|
|
||||||
|
|
||||||
b.ResetTimer()
|
|
||||||
|
|
||||||
var hit, miss int
|
|
||||||
for i := 0; i < 2*b.N; i++ {
|
|
||||||
if i%2 == 0 {
|
|
||||||
l.Add(trace[i], trace[i])
|
|
||||||
} else {
|
|
||||||
_, ok := l.Get(trace[i])
|
|
||||||
if ok {
|
|
||||||
hit++
|
|
||||||
} else {
|
|
||||||
miss++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
b.Logf("hit: %d miss: %d ratio: %f", hit, miss, float64(hit)/float64(miss))
|
|
||||||
}
|
|
||||||
|
|
||||||
func Benchmark2Q_Freq(b *testing.B) {
|
|
||||||
l, err := New2Q(8192)
|
|
||||||
if err != nil {
|
|
||||||
b.Fatalf("err: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
trace := make([]int64, b.N*2)
|
|
||||||
for i := 0; i < b.N*2; i++ {
|
|
||||||
if i%2 == 0 {
|
|
||||||
trace[i] = rand.Int63() % 16384
|
|
||||||
} else {
|
|
||||||
trace[i] = rand.Int63() % 32768
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
b.ResetTimer()
|
|
||||||
|
|
||||||
for i := 0; i < b.N; i++ {
|
|
||||||
l.Add(trace[i], trace[i])
|
|
||||||
}
|
|
||||||
var hit, miss int
|
|
||||||
for i := 0; i < b.N; i++ {
|
|
||||||
_, ok := l.Get(trace[i])
|
|
||||||
if ok {
|
|
||||||
hit++
|
|
||||||
} else {
|
|
||||||
miss++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
b.Logf("hit: %d miss: %d ratio: %f", hit, miss, float64(hit)/float64(miss))
|
|
||||||
}
|
|
||||||
|
|
||||||
func Test2Q_RandomOps(t *testing.T) {
|
|
||||||
size := 128
|
|
||||||
l, err := New2Q(128)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("err: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
n := 200000
|
|
||||||
for i := 0; i < n; i++ {
|
|
||||||
key := rand.Int63() % 512
|
|
||||||
r := rand.Int63()
|
|
||||||
switch r % 3 {
|
|
||||||
case 0:
|
|
||||||
l.Add(key, key)
|
|
||||||
case 1:
|
|
||||||
l.Get(key)
|
|
||||||
case 2:
|
|
||||||
l.Remove(key)
|
|
||||||
}
|
|
||||||
|
|
||||||
if l.recent.Len()+l.frequent.Len() > size {
|
|
||||||
t.Fatalf("bad: recent: %d freq: %d",
|
|
||||||
l.recent.Len(), l.frequent.Len())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func Test2Q_Get_RecentToFrequent(t *testing.T) {
|
|
||||||
l, err := New2Q(128)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("err: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Touch all the entries, should be in t1
|
|
||||||
for i := 0; i < 128; i++ {
|
|
||||||
l.Add(i, i)
|
|
||||||
}
|
|
||||||
if n := l.recent.Len(); n != 128 {
|
|
||||||
t.Fatalf("bad: %d", n)
|
|
||||||
}
|
|
||||||
if n := l.frequent.Len(); n != 0 {
|
|
||||||
t.Fatalf("bad: %d", n)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get should upgrade to t2
|
|
||||||
for i := 0; i < 128; i++ {
|
|
||||||
_, ok := l.Get(i)
|
|
||||||
if !ok {
|
|
||||||
t.Fatalf("missing: %d", i)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if n := l.recent.Len(); n != 0 {
|
|
||||||
t.Fatalf("bad: %d", n)
|
|
||||||
}
|
|
||||||
if n := l.frequent.Len(); n != 128 {
|
|
||||||
t.Fatalf("bad: %d", n)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get be from t2
|
|
||||||
for i := 0; i < 128; i++ {
|
|
||||||
_, ok := l.Get(i)
|
|
||||||
if !ok {
|
|
||||||
t.Fatalf("missing: %d", i)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if n := l.recent.Len(); n != 0 {
|
|
||||||
t.Fatalf("bad: %d", n)
|
|
||||||
}
|
|
||||||
if n := l.frequent.Len(); n != 128 {
|
|
||||||
t.Fatalf("bad: %d", n)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func Test2Q_Add_RecentToFrequent(t *testing.T) {
|
|
||||||
l, err := New2Q(128)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("err: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add initially to recent
|
|
||||||
l.Add(1, 1)
|
|
||||||
if n := l.recent.Len(); n != 1 {
|
|
||||||
t.Fatalf("bad: %d", n)
|
|
||||||
}
|
|
||||||
if n := l.frequent.Len(); n != 0 {
|
|
||||||
t.Fatalf("bad: %d", n)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add should upgrade to frequent
|
|
||||||
l.Add(1, 1)
|
|
||||||
if n := l.recent.Len(); n != 0 {
|
|
||||||
t.Fatalf("bad: %d", n)
|
|
||||||
}
|
|
||||||
if n := l.frequent.Len(); n != 1 {
|
|
||||||
t.Fatalf("bad: %d", n)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add should remain in frequent
|
|
||||||
l.Add(1, 1)
|
|
||||||
if n := l.recent.Len(); n != 0 {
|
|
||||||
t.Fatalf("bad: %d", n)
|
|
||||||
}
|
|
||||||
if n := l.frequent.Len(); n != 1 {
|
|
||||||
t.Fatalf("bad: %d", n)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func Test2Q_Add_RecentEvict(t *testing.T) {
|
|
||||||
l, err := New2Q(4)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("err: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add 1,2,3,4,5 -> Evict 1
|
|
||||||
l.Add(1, 1)
|
|
||||||
l.Add(2, 2)
|
|
||||||
l.Add(3, 3)
|
|
||||||
l.Add(4, 4)
|
|
||||||
l.Add(5, 5)
|
|
||||||
if n := l.recent.Len(); n != 4 {
|
|
||||||
t.Fatalf("bad: %d", n)
|
|
||||||
}
|
|
||||||
if n := l.recentEvict.Len(); n != 1 {
|
|
||||||
t.Fatalf("bad: %d", n)
|
|
||||||
}
|
|
||||||
if n := l.frequent.Len(); n != 0 {
|
|
||||||
t.Fatalf("bad: %d", n)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Pull in the recently evicted
|
|
||||||
l.Add(1, 1)
|
|
||||||
if n := l.recent.Len(); n != 3 {
|
|
||||||
t.Fatalf("bad: %d", n)
|
|
||||||
}
|
|
||||||
if n := l.recentEvict.Len(); n != 1 {
|
|
||||||
t.Fatalf("bad: %d", n)
|
|
||||||
}
|
|
||||||
if n := l.frequent.Len(); n != 1 {
|
|
||||||
t.Fatalf("bad: %d", n)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add 6, should cause another recent evict
|
|
||||||
l.Add(6, 6)
|
|
||||||
if n := l.recent.Len(); n != 3 {
|
|
||||||
t.Fatalf("bad: %d", n)
|
|
||||||
}
|
|
||||||
if n := l.recentEvict.Len(); n != 2 {
|
|
||||||
t.Fatalf("bad: %d", n)
|
|
||||||
}
|
|
||||||
if n := l.frequent.Len(); n != 1 {
|
|
||||||
t.Fatalf("bad: %d", n)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func Test2Q(t *testing.T) {
|
|
||||||
l, err := New2Q(128)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("err: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
for i := 0; i < 256; i++ {
|
|
||||||
l.Add(i, i)
|
|
||||||
}
|
|
||||||
if l.Len() != 128 {
|
|
||||||
t.Fatalf("bad len: %v", l.Len())
|
|
||||||
}
|
|
||||||
|
|
||||||
for i, k := range l.Keys() {
|
|
||||||
if v, ok := l.Get(k); !ok || v != k || v != i+128 {
|
|
||||||
t.Fatalf("bad key: %v", k)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for i := 0; i < 128; i++ {
|
|
||||||
_, ok := l.Get(i)
|
|
||||||
if ok {
|
|
||||||
t.Fatalf("should be evicted")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for i := 128; i < 256; i++ {
|
|
||||||
_, ok := l.Get(i)
|
|
||||||
if !ok {
|
|
||||||
t.Fatalf("should not be evicted")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for i := 128; i < 192; i++ {
|
|
||||||
l.Remove(i)
|
|
||||||
_, ok := l.Get(i)
|
|
||||||
if ok {
|
|
||||||
t.Fatalf("should be deleted")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
l.Purge()
|
|
||||||
if l.Len() != 0 {
|
|
||||||
t.Fatalf("bad len: %v", l.Len())
|
|
||||||
}
|
|
||||||
if _, ok := l.Get(200); ok {
|
|
||||||
t.Fatalf("should contain nothing")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Test that Contains doesn't update recent-ness
|
|
||||||
func Test2Q_Contains(t *testing.T) {
|
|
||||||
l, err := New2Q(2)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("err: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
l.Add(1, 1)
|
|
||||||
l.Add(2, 2)
|
|
||||||
if !l.Contains(1) {
|
|
||||||
t.Errorf("1 should be contained")
|
|
||||||
}
|
|
||||||
|
|
||||||
l.Add(3, 3)
|
|
||||||
if l.Contains(1) {
|
|
||||||
t.Errorf("Contains should not have updated recent-ness of 1")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Test that Peek doesn't update recent-ness
|
|
||||||
func Test2Q_Peek(t *testing.T) {
|
|
||||||
l, err := New2Q(2)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("err: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
l.Add(1, 1)
|
|
||||||
l.Add(2, 2)
|
|
||||||
if v, ok := l.Peek(1); !ok || v != 1 {
|
|
||||||
t.Errorf("1 should be set to 1: %v, %v", v, ok)
|
|
||||||
}
|
|
||||||
|
|
||||||
l.Add(3, 3)
|
|
||||||
if l.Contains(1) {
|
|
||||||
t.Errorf("should not have updated recent-ness of 1")
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,362 +0,0 @@
|
||||||
Mozilla Public License, version 2.0
|
|
||||||
|
|
||||||
1. Definitions
|
|
||||||
|
|
||||||
1.1. "Contributor"
|
|
||||||
|
|
||||||
means each individual or legal entity that creates, contributes to the
|
|
||||||
creation of, or owns Covered Software.
|
|
||||||
|
|
||||||
1.2. "Contributor Version"
|
|
||||||
|
|
||||||
means the combination of the Contributions of others (if any) used by a
|
|
||||||
Contributor and that particular Contributor's Contribution.
|
|
||||||
|
|
||||||
1.3. "Contribution"
|
|
||||||
|
|
||||||
means Covered Software of a particular Contributor.
|
|
||||||
|
|
||||||
1.4. "Covered Software"
|
|
||||||
|
|
||||||
means Source Code Form to which the initial Contributor has attached the
|
|
||||||
notice in Exhibit A, the Executable Form of such Source Code Form, and
|
|
||||||
Modifications of such Source Code Form, in each case including portions
|
|
||||||
thereof.
|
|
||||||
|
|
||||||
1.5. "Incompatible With Secondary Licenses"
|
|
||||||
means
|
|
||||||
|
|
||||||
a. that the initial Contributor has attached the notice described in
|
|
||||||
Exhibit B to the Covered Software; or
|
|
||||||
|
|
||||||
b. that the Covered Software was made available under the terms of
|
|
||||||
version 1.1 or earlier of the License, but not also under the terms of
|
|
||||||
a Secondary License.
|
|
||||||
|
|
||||||
1.6. "Executable Form"
|
|
||||||
|
|
||||||
means any form of the work other than Source Code Form.
|
|
||||||
|
|
||||||
1.7. "Larger Work"
|
|
||||||
|
|
||||||
means a work that combines Covered Software with other material, in a
|
|
||||||
separate file or files, that is not Covered Software.
|
|
||||||
|
|
||||||
1.8. "License"
|
|
||||||
|
|
||||||
means this document.
|
|
||||||
|
|
||||||
1.9. "Licensable"
|
|
||||||
|
|
||||||
means having the right to grant, to the maximum extent possible, whether
|
|
||||||
at the time of the initial grant or subsequently, any and all of the
|
|
||||||
rights conveyed by this License.
|
|
||||||
|
|
||||||
1.10. "Modifications"
|
|
||||||
|
|
||||||
means any of the following:
|
|
||||||
|
|
||||||
a. any file in Source Code Form that results from an addition to,
|
|
||||||
deletion from, or modification of the contents of Covered Software; or
|
|
||||||
|
|
||||||
b. any new file in Source Code Form that contains any Covered Software.
|
|
||||||
|
|
||||||
1.11. "Patent Claims" of a Contributor
|
|
||||||
|
|
||||||
means any patent claim(s), including without limitation, method,
|
|
||||||
process, and apparatus claims, in any patent Licensable by such
|
|
||||||
Contributor that would be infringed, but for the grant of the License,
|
|
||||||
by the making, using, selling, offering for sale, having made, import,
|
|
||||||
or transfer of either its Contributions or its Contributor Version.
|
|
||||||
|
|
||||||
1.12. "Secondary License"
|
|
||||||
|
|
||||||
means either the GNU General Public License, Version 2.0, the GNU Lesser
|
|
||||||
General Public License, Version 2.1, the GNU Affero General Public
|
|
||||||
License, Version 3.0, or any later versions of those licenses.
|
|
||||||
|
|
||||||
1.13. "Source Code Form"
|
|
||||||
|
|
||||||
means the form of the work preferred for making modifications.
|
|
||||||
|
|
||||||
1.14. "You" (or "Your")
|
|
||||||
|
|
||||||
means an individual or a legal entity exercising rights under this
|
|
||||||
License. For legal entities, "You" includes any entity that controls, is
|
|
||||||
controlled by, or is under common control with You. For purposes of this
|
|
||||||
definition, "control" means (a) the power, direct or indirect, to cause
|
|
||||||
the direction or management of such entity, whether by contract or
|
|
||||||
otherwise, or (b) ownership of more than fifty percent (50%) of the
|
|
||||||
outstanding shares or beneficial ownership of such entity.
|
|
||||||
|
|
||||||
|
|
||||||
2. License Grants and Conditions
|
|
||||||
|
|
||||||
2.1. Grants
|
|
||||||
|
|
||||||
Each Contributor hereby grants You a world-wide, royalty-free,
|
|
||||||
non-exclusive license:
|
|
||||||
|
|
||||||
a. under intellectual property rights (other than patent or trademark)
|
|
||||||
Licensable by such Contributor to use, reproduce, make available,
|
|
||||||
modify, display, perform, distribute, and otherwise exploit its
|
|
||||||
Contributions, either on an unmodified basis, with Modifications, or
|
|
||||||
as part of a Larger Work; and
|
|
||||||
|
|
||||||
b. under Patent Claims of such Contributor to make, use, sell, offer for
|
|
||||||
sale, have made, import, and otherwise transfer either its
|
|
||||||
Contributions or its Contributor Version.
|
|
||||||
|
|
||||||
2.2. Effective Date
|
|
||||||
|
|
||||||
The licenses granted in Section 2.1 with respect to any Contribution
|
|
||||||
become effective for each Contribution on the date the Contributor first
|
|
||||||
distributes such Contribution.
|
|
||||||
|
|
||||||
2.3. Limitations on Grant Scope
|
|
||||||
|
|
||||||
The licenses granted in this Section 2 are the only rights granted under
|
|
||||||
this License. No additional rights or licenses will be implied from the
|
|
||||||
distribution or licensing of Covered Software under this License.
|
|
||||||
Notwithstanding Section 2.1(b) above, no patent license is granted by a
|
|
||||||
Contributor:
|
|
||||||
|
|
||||||
a. for any code that a Contributor has removed from Covered Software; or
|
|
||||||
|
|
||||||
b. for infringements caused by: (i) Your and any other third party's
|
|
||||||
modifications of Covered Software, or (ii) the combination of its
|
|
||||||
Contributions with other software (except as part of its Contributor
|
|
||||||
Version); or
|
|
||||||
|
|
||||||
c. under Patent Claims infringed by Covered Software in the absence of
|
|
||||||
its Contributions.
|
|
||||||
|
|
||||||
This License does not grant any rights in the trademarks, service marks,
|
|
||||||
or logos of any Contributor (except as may be necessary to comply with
|
|
||||||
the notice requirements in Section 3.4).
|
|
||||||
|
|
||||||
2.4. Subsequent Licenses
|
|
||||||
|
|
||||||
No Contributor makes additional grants as a result of Your choice to
|
|
||||||
distribute the Covered Software under a subsequent version of this
|
|
||||||
License (see Section 10.2) or under the terms of a Secondary License (if
|
|
||||||
permitted under the terms of Section 3.3).
|
|
||||||
|
|
||||||
2.5. Representation
|
|
||||||
|
|
||||||
Each Contributor represents that the Contributor believes its
|
|
||||||
Contributions are its original creation(s) or it has sufficient rights to
|
|
||||||
grant the rights to its Contributions conveyed by this License.
|
|
||||||
|
|
||||||
2.6. Fair Use
|
|
||||||
|
|
||||||
This License is not intended to limit any rights You have under
|
|
||||||
applicable copyright doctrines of fair use, fair dealing, or other
|
|
||||||
equivalents.
|
|
||||||
|
|
||||||
2.7. Conditions
|
|
||||||
|
|
||||||
Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in
|
|
||||||
Section 2.1.
|
|
||||||
|
|
||||||
|
|
||||||
3. Responsibilities
|
|
||||||
|
|
||||||
3.1. Distribution of Source Form
|
|
||||||
|
|
||||||
All distribution of Covered Software in Source Code Form, including any
|
|
||||||
Modifications that You create or to which You contribute, must be under
|
|
||||||
the terms of this License. You must inform recipients that the Source
|
|
||||||
Code Form of the Covered Software is governed by the terms of this
|
|
||||||
License, and how they can obtain a copy of this License. You may not
|
|
||||||
attempt to alter or restrict the recipients' rights in the Source Code
|
|
||||||
Form.
|
|
||||||
|
|
||||||
3.2. Distribution of Executable Form
|
|
||||||
|
|
||||||
If You distribute Covered Software in Executable Form then:
|
|
||||||
|
|
||||||
a. such Covered Software must also be made available in Source Code Form,
|
|
||||||
as described in Section 3.1, and You must inform recipients of the
|
|
||||||
Executable Form how they can obtain a copy of such Source Code Form by
|
|
||||||
reasonable means in a timely manner, at a charge no more than the cost
|
|
||||||
of distribution to the recipient; and
|
|
||||||
|
|
||||||
b. You may distribute such Executable Form under the terms of this
|
|
||||||
License, or sublicense it under different terms, provided that the
|
|
||||||
license for the Executable Form does not attempt to limit or alter the
|
|
||||||
recipients' rights in the Source Code Form under this License.
|
|
||||||
|
|
||||||
3.3. Distribution of a Larger Work
|
|
||||||
|
|
||||||
You may create and distribute a Larger Work under terms of Your choice,
|
|
||||||
provided that You also comply with the requirements of this License for
|
|
||||||
the Covered Software. If the Larger Work is a combination of Covered
|
|
||||||
Software with a work governed by one or more Secondary Licenses, and the
|
|
||||||
Covered Software is not Incompatible With Secondary Licenses, this
|
|
||||||
License permits You to additionally distribute such Covered Software
|
|
||||||
under the terms of such Secondary License(s), so that the recipient of
|
|
||||||
the Larger Work may, at their option, further distribute the Covered
|
|
||||||
Software under the terms of either this License or such Secondary
|
|
||||||
License(s).
|
|
||||||
|
|
||||||
3.4. Notices
|
|
||||||
|
|
||||||
You may not remove or alter the substance of any license notices
|
|
||||||
(including copyright notices, patent notices, disclaimers of warranty, or
|
|
||||||
limitations of liability) contained within the Source Code Form of the
|
|
||||||
Covered Software, except that You may alter any license notices to the
|
|
||||||
extent required to remedy known factual inaccuracies.
|
|
||||||
|
|
||||||
3.5. Application of Additional Terms
|
|
||||||
|
|
||||||
You may choose to offer, and to charge a fee for, warranty, support,
|
|
||||||
indemnity or liability obligations to one or more recipients of Covered
|
|
||||||
Software. However, You may do so only on Your own behalf, and not on
|
|
||||||
behalf of any Contributor. You must make it absolutely clear that any
|
|
||||||
such warranty, support, indemnity, or liability obligation is offered by
|
|
||||||
You alone, and You hereby agree to indemnify every Contributor for any
|
|
||||||
liability incurred by such Contributor as a result of warranty, support,
|
|
||||||
indemnity or liability terms You offer. You may include additional
|
|
||||||
disclaimers of warranty and limitations of liability specific to any
|
|
||||||
jurisdiction.
|
|
||||||
|
|
||||||
4. Inability to Comply Due to Statute or Regulation
|
|
||||||
|
|
||||||
If it is impossible for You to comply with any of the terms of this License
|
|
||||||
with respect to some or all of the Covered Software due to statute,
|
|
||||||
judicial order, or regulation then You must: (a) comply with the terms of
|
|
||||||
this License to the maximum extent possible; and (b) describe the
|
|
||||||
limitations and the code they affect. Such description must be placed in a
|
|
||||||
text file included with all distributions of the Covered Software under
|
|
||||||
this License. Except to the extent prohibited by statute or regulation,
|
|
||||||
such description must be sufficiently detailed for a recipient of ordinary
|
|
||||||
skill to be able to understand it.
|
|
||||||
|
|
||||||
5. Termination
|
|
||||||
|
|
||||||
5.1. The rights granted under this License will terminate automatically if You
|
|
||||||
fail to comply with any of its terms. However, if You become compliant,
|
|
||||||
then the rights granted under this License from a particular Contributor
|
|
||||||
are reinstated (a) provisionally, unless and until such Contributor
|
|
||||||
explicitly and finally terminates Your grants, and (b) on an ongoing
|
|
||||||
basis, if such Contributor fails to notify You of the non-compliance by
|
|
||||||
some reasonable means prior to 60 days after You have come back into
|
|
||||||
compliance. Moreover, Your grants from a particular Contributor are
|
|
||||||
reinstated on an ongoing basis if such Contributor notifies You of the
|
|
||||||
non-compliance by some reasonable means, this is the first time You have
|
|
||||||
received notice of non-compliance with this License from such
|
|
||||||
Contributor, and You become compliant prior to 30 days after Your receipt
|
|
||||||
of the notice.
|
|
||||||
|
|
||||||
5.2. If You initiate litigation against any entity by asserting a patent
|
|
||||||
infringement claim (excluding declaratory judgment actions,
|
|
||||||
counter-claims, and cross-claims) alleging that a Contributor Version
|
|
||||||
directly or indirectly infringes any patent, then the rights granted to
|
|
||||||
You by any and all Contributors for the Covered Software under Section
|
|
||||||
2.1 of this License shall terminate.
|
|
||||||
|
|
||||||
5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user
|
|
||||||
license agreements (excluding distributors and resellers) which have been
|
|
||||||
validly granted by You or Your distributors under this License prior to
|
|
||||||
termination shall survive termination.
|
|
||||||
|
|
||||||
6. Disclaimer of Warranty
|
|
||||||
|
|
||||||
Covered Software is provided under this License on an "as is" basis,
|
|
||||||
without warranty of any kind, either expressed, implied, or statutory,
|
|
||||||
including, without limitation, warranties that the Covered Software is free
|
|
||||||
of defects, merchantable, fit for a particular purpose or non-infringing.
|
|
||||||
The entire risk as to the quality and performance of the Covered Software
|
|
||||||
is with You. Should any Covered Software prove defective in any respect,
|
|
||||||
You (not any Contributor) assume the cost of any necessary servicing,
|
|
||||||
repair, or correction. This disclaimer of warranty constitutes an essential
|
|
||||||
part of this License. No use of any Covered Software is authorized under
|
|
||||||
this License except under this disclaimer.
|
|
||||||
|
|
||||||
7. Limitation of Liability
|
|
||||||
|
|
||||||
Under no circumstances and under no legal theory, whether tort (including
|
|
||||||
negligence), contract, or otherwise, shall any Contributor, or anyone who
|
|
||||||
distributes Covered Software as permitted above, be liable to You for any
|
|
||||||
direct, indirect, special, incidental, or consequential damages of any
|
|
||||||
character including, without limitation, damages for lost profits, loss of
|
|
||||||
goodwill, work stoppage, computer failure or malfunction, or any and all
|
|
||||||
other commercial damages or losses, even if such party shall have been
|
|
||||||
informed of the possibility of such damages. This limitation of liability
|
|
||||||
shall not apply to liability for death or personal injury resulting from
|
|
||||||
such party's negligence to the extent applicable law prohibits such
|
|
||||||
limitation. Some jurisdictions do not allow the exclusion or limitation of
|
|
||||||
incidental or consequential damages, so this exclusion and limitation may
|
|
||||||
not apply to You.
|
|
||||||
|
|
||||||
8. Litigation
|
|
||||||
|
|
||||||
Any litigation relating to this License may be brought only in the courts
|
|
||||||
of a jurisdiction where the defendant maintains its principal place of
|
|
||||||
business and such litigation shall be governed by laws of that
|
|
||||||
jurisdiction, without reference to its conflict-of-law provisions. Nothing
|
|
||||||
in this Section shall prevent a party's ability to bring cross-claims or
|
|
||||||
counter-claims.
|
|
||||||
|
|
||||||
9. Miscellaneous
|
|
||||||
|
|
||||||
This License represents the complete agreement concerning the subject
|
|
||||||
matter hereof. If any provision of this License is held to be
|
|
||||||
unenforceable, such provision shall be reformed only to the extent
|
|
||||||
necessary to make it enforceable. Any law or regulation which provides that
|
|
||||||
the language of a contract shall be construed against the drafter shall not
|
|
||||||
be used to construe this License against a Contributor.
|
|
||||||
|
|
||||||
|
|
||||||
10. Versions of the License
|
|
||||||
|
|
||||||
10.1. New Versions
|
|
||||||
|
|
||||||
Mozilla Foundation is the license steward. Except as provided in Section
|
|
||||||
10.3, no one other than the license steward has the right to modify or
|
|
||||||
publish new versions of this License. Each version will be given a
|
|
||||||
distinguishing version number.
|
|
||||||
|
|
||||||
10.2. Effect of New Versions
|
|
||||||
|
|
||||||
You may distribute the Covered Software under the terms of the version
|
|
||||||
of the License under which You originally received the Covered Software,
|
|
||||||
or under the terms of any subsequent version published by the license
|
|
||||||
steward.
|
|
||||||
|
|
||||||
10.3. Modified Versions
|
|
||||||
|
|
||||||
If you create software not governed by this License, and you want to
|
|
||||||
create a new license for such software, you may create and use a
|
|
||||||
modified version of this License if you rename the license and remove
|
|
||||||
any references to the name of the license steward (except to note that
|
|
||||||
such modified license differs from this License).
|
|
||||||
|
|
||||||
10.4. Distributing Source Code Form that is Incompatible With Secondary
|
|
||||||
Licenses If You choose to distribute Source Code Form that is
|
|
||||||
Incompatible With Secondary Licenses under the terms of this version of
|
|
||||||
the License, the notice described in Exhibit B of this License must be
|
|
||||||
attached.
|
|
||||||
|
|
||||||
Exhibit A - Source Code Form License Notice
|
|
||||||
|
|
||||||
This Source Code Form is subject to the
|
|
||||||
terms of the Mozilla Public License, v.
|
|
||||||
2.0. If a copy of the MPL was not
|
|
||||||
distributed with this file, You can
|
|
||||||
obtain one at
|
|
||||||
http://mozilla.org/MPL/2.0/.
|
|
||||||
|
|
||||||
If it is not possible or desirable to put the notice in a particular file,
|
|
||||||
then You may include the notice in a location (such as a LICENSE file in a
|
|
||||||
relevant directory) where a recipient would be likely to look for such a
|
|
||||||
notice.
|
|
||||||
|
|
||||||
You may add additional accurate notices of copyright ownership.
|
|
||||||
|
|
||||||
Exhibit B - "Incompatible With Secondary Licenses" Notice
|
|
||||||
|
|
||||||
This Source Code Form is "Incompatible
|
|
||||||
With Secondary Licenses", as defined by
|
|
||||||
the Mozilla Public License, v. 2.0.
|
|
|
@ -1,25 +0,0 @@
|
||||||
golang-lru
|
|
||||||
==========
|
|
||||||
|
|
||||||
This provides the `lru` package which implements a fixed-size
|
|
||||||
thread safe LRU cache. It is based on the cache in Groupcache.
|
|
||||||
|
|
||||||
Documentation
|
|
||||||
=============
|
|
||||||
|
|
||||||
Full docs are available on [Godoc](http://godoc.org/github.com/hashicorp/golang-lru)
|
|
||||||
|
|
||||||
Example
|
|
||||||
=======
|
|
||||||
|
|
||||||
Using the LRU is very simple:
|
|
||||||
|
|
||||||
```go
|
|
||||||
l, _ := New(128)
|
|
||||||
for i := 0; i < 256; i++ {
|
|
||||||
l.Add(i, nil)
|
|
||||||
}
|
|
||||||
if l.Len() != 128 {
|
|
||||||
panic(fmt.Sprintf("bad len: %v", l.Len()))
|
|
||||||
}
|
|
||||||
```
|
|
|
@ -1,257 +0,0 @@
|
||||||
package lru
|
|
||||||
|
|
||||||
import (
|
|
||||||
"sync"
|
|
||||||
|
|
||||||
"github.com/hashicorp/golang-lru/simplelru"
|
|
||||||
)
|
|
||||||
|
|
||||||
// ARCCache is a thread-safe fixed size Adaptive Replacement Cache (ARC).
|
|
||||||
// ARC is an enhancement over the standard LRU cache in that tracks both
|
|
||||||
// frequency and recency of use. This avoids a burst in access to new
|
|
||||||
// entries from evicting the frequently used older entries. It adds some
|
|
||||||
// additional tracking overhead to a standard LRU cache, computationally
|
|
||||||
// it is roughly 2x the cost, and the extra memory overhead is linear
|
|
||||||
// with the size of the cache. ARC has been patented by IBM, but is
|
|
||||||
// similar to the TwoQueueCache (2Q) which requires setting parameters.
|
|
||||||
type ARCCache struct {
|
|
||||||
size int // Size is the total capacity of the cache
|
|
||||||
p int // P is the dynamic preference towards T1 or T2
|
|
||||||
|
|
||||||
t1 *simplelru.LRU // T1 is the LRU for recently accessed items
|
|
||||||
b1 *simplelru.LRU // B1 is the LRU for evictions from t1
|
|
||||||
|
|
||||||
t2 *simplelru.LRU // T2 is the LRU for frequently accessed items
|
|
||||||
b2 *simplelru.LRU // B2 is the LRU for evictions from t2
|
|
||||||
|
|
||||||
lock sync.RWMutex
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewARC creates an ARC of the given size
|
|
||||||
func NewARC(size int) (*ARCCache, error) {
|
|
||||||
// Create the sub LRUs
|
|
||||||
b1, err := simplelru.NewLRU(size, nil)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
b2, err := simplelru.NewLRU(size, nil)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
t1, err := simplelru.NewLRU(size, nil)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
t2, err := simplelru.NewLRU(size, nil)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// Initialize the ARC
|
|
||||||
c := &ARCCache{
|
|
||||||
size: size,
|
|
||||||
p: 0,
|
|
||||||
t1: t1,
|
|
||||||
b1: b1,
|
|
||||||
t2: t2,
|
|
||||||
b2: b2,
|
|
||||||
}
|
|
||||||
return c, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get looks up a key's value from the cache.
|
|
||||||
func (c *ARCCache) Get(key interface{}) (interface{}, bool) {
|
|
||||||
c.lock.Lock()
|
|
||||||
defer c.lock.Unlock()
|
|
||||||
|
|
||||||
// Ff the value is contained in T1 (recent), then
|
|
||||||
// promote it to T2 (frequent)
|
|
||||||
if val, ok := c.t1.Peek(key); ok {
|
|
||||||
c.t1.Remove(key)
|
|
||||||
c.t2.Add(key, val)
|
|
||||||
return val, ok
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check if the value is contained in T2 (frequent)
|
|
||||||
if val, ok := c.t2.Get(key); ok {
|
|
||||||
return val, ok
|
|
||||||
}
|
|
||||||
|
|
||||||
// No hit
|
|
||||||
return nil, false
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add adds a value to the cache.
|
|
||||||
func (c *ARCCache) Add(key, value interface{}) {
|
|
||||||
c.lock.Lock()
|
|
||||||
defer c.lock.Unlock()
|
|
||||||
|
|
||||||
// Check if the value is contained in T1 (recent), and potentially
|
|
||||||
// promote it to frequent T2
|
|
||||||
if c.t1.Contains(key) {
|
|
||||||
c.t1.Remove(key)
|
|
||||||
c.t2.Add(key, value)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check if the value is already in T2 (frequent) and update it
|
|
||||||
if c.t2.Contains(key) {
|
|
||||||
c.t2.Add(key, value)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check if this value was recently evicted as part of the
|
|
||||||
// recently used list
|
|
||||||
if c.b1.Contains(key) {
|
|
||||||
// T1 set is too small, increase P appropriately
|
|
||||||
delta := 1
|
|
||||||
b1Len := c.b1.Len()
|
|
||||||
b2Len := c.b2.Len()
|
|
||||||
if b2Len > b1Len {
|
|
||||||
delta = b2Len / b1Len
|
|
||||||
}
|
|
||||||
if c.p+delta >= c.size {
|
|
||||||
c.p = c.size
|
|
||||||
} else {
|
|
||||||
c.p += delta
|
|
||||||
}
|
|
||||||
|
|
||||||
// Potentially need to make room in the cache
|
|
||||||
if c.t1.Len()+c.t2.Len() >= c.size {
|
|
||||||
c.replace(false)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Remove from B1
|
|
||||||
c.b1.Remove(key)
|
|
||||||
|
|
||||||
// Add the key to the frequently used list
|
|
||||||
c.t2.Add(key, value)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check if this value was recently evicted as part of the
|
|
||||||
// frequently used list
|
|
||||||
if c.b2.Contains(key) {
|
|
||||||
// T2 set is too small, decrease P appropriately
|
|
||||||
delta := 1
|
|
||||||
b1Len := c.b1.Len()
|
|
||||||
b2Len := c.b2.Len()
|
|
||||||
if b1Len > b2Len {
|
|
||||||
delta = b1Len / b2Len
|
|
||||||
}
|
|
||||||
if delta >= c.p {
|
|
||||||
c.p = 0
|
|
||||||
} else {
|
|
||||||
c.p -= delta
|
|
||||||
}
|
|
||||||
|
|
||||||
// Potentially need to make room in the cache
|
|
||||||
if c.t1.Len()+c.t2.Len() >= c.size {
|
|
||||||
c.replace(true)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Remove from B2
|
|
||||||
c.b2.Remove(key)
|
|
||||||
|
|
||||||
// Add the key to the frequntly used list
|
|
||||||
c.t2.Add(key, value)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Potentially need to make room in the cache
|
|
||||||
if c.t1.Len()+c.t2.Len() >= c.size {
|
|
||||||
c.replace(false)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Keep the size of the ghost buffers trim
|
|
||||||
if c.b1.Len() > c.size-c.p {
|
|
||||||
c.b1.RemoveOldest()
|
|
||||||
}
|
|
||||||
if c.b2.Len() > c.p {
|
|
||||||
c.b2.RemoveOldest()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add to the recently seen list
|
|
||||||
c.t1.Add(key, value)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// replace is used to adaptively evict from either T1 or T2
|
|
||||||
// based on the current learned value of P
|
|
||||||
func (c *ARCCache) replace(b2ContainsKey bool) {
|
|
||||||
t1Len := c.t1.Len()
|
|
||||||
if t1Len > 0 && (t1Len > c.p || (t1Len == c.p && b2ContainsKey)) {
|
|
||||||
k, _, ok := c.t1.RemoveOldest()
|
|
||||||
if ok {
|
|
||||||
c.b1.Add(k, nil)
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
k, _, ok := c.t2.RemoveOldest()
|
|
||||||
if ok {
|
|
||||||
c.b2.Add(k, nil)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Len returns the number of cached entries
|
|
||||||
func (c *ARCCache) Len() int {
|
|
||||||
c.lock.RLock()
|
|
||||||
defer c.lock.RUnlock()
|
|
||||||
return c.t1.Len() + c.t2.Len()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Keys returns all the cached keys
|
|
||||||
func (c *ARCCache) Keys() []interface{} {
|
|
||||||
c.lock.RLock()
|
|
||||||
defer c.lock.RUnlock()
|
|
||||||
k1 := c.t1.Keys()
|
|
||||||
k2 := c.t2.Keys()
|
|
||||||
return append(k1, k2...)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Remove is used to purge a key from the cache
|
|
||||||
func (c *ARCCache) Remove(key interface{}) {
|
|
||||||
c.lock.Lock()
|
|
||||||
defer c.lock.Unlock()
|
|
||||||
if c.t1.Remove(key) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if c.t2.Remove(key) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if c.b1.Remove(key) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
if c.b2.Remove(key) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Purge is used to clear the cache
|
|
||||||
func (c *ARCCache) Purge() {
|
|
||||||
c.lock.Lock()
|
|
||||||
defer c.lock.Unlock()
|
|
||||||
c.t1.Purge()
|
|
||||||
c.t2.Purge()
|
|
||||||
c.b1.Purge()
|
|
||||||
c.b2.Purge()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Contains is used to check if the cache contains a key
|
|
||||||
// without updating recency or frequency.
|
|
||||||
func (c *ARCCache) Contains(key interface{}) bool {
|
|
||||||
c.lock.RLock()
|
|
||||||
defer c.lock.RUnlock()
|
|
||||||
return c.t1.Contains(key) || c.t2.Contains(key)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Peek is used to inspect the cache value of a key
|
|
||||||
// without updating recency or frequency.
|
|
||||||
func (c *ARCCache) Peek(key interface{}) (interface{}, bool) {
|
|
||||||
c.lock.RLock()
|
|
||||||
defer c.lock.RUnlock()
|
|
||||||
if val, ok := c.t1.Peek(key); ok {
|
|
||||||
return val, ok
|
|
||||||
}
|
|
||||||
return c.t2.Peek(key)
|
|
||||||
}
|
|
|
@ -1,377 +0,0 @@
|
||||||
package lru
|
|
||||||
|
|
||||||
import (
|
|
||||||
"math/rand"
|
|
||||||
"testing"
|
|
||||||
"time"
|
|
||||||
)
|
|
||||||
|
|
||||||
func init() {
|
|
||||||
rand.Seed(time.Now().Unix())
|
|
||||||
}
|
|
||||||
|
|
||||||
func BenchmarkARC_Rand(b *testing.B) {
|
|
||||||
l, err := NewARC(8192)
|
|
||||||
if err != nil {
|
|
||||||
b.Fatalf("err: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
trace := make([]int64, b.N*2)
|
|
||||||
for i := 0; i < b.N*2; i++ {
|
|
||||||
trace[i] = rand.Int63() % 32768
|
|
||||||
}
|
|
||||||
|
|
||||||
b.ResetTimer()
|
|
||||||
|
|
||||||
var hit, miss int
|
|
||||||
for i := 0; i < 2*b.N; i++ {
|
|
||||||
if i%2 == 0 {
|
|
||||||
l.Add(trace[i], trace[i])
|
|
||||||
} else {
|
|
||||||
_, ok := l.Get(trace[i])
|
|
||||||
if ok {
|
|
||||||
hit++
|
|
||||||
} else {
|
|
||||||
miss++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
b.Logf("hit: %d miss: %d ratio: %f", hit, miss, float64(hit)/float64(miss))
|
|
||||||
}
|
|
||||||
|
|
||||||
func BenchmarkARC_Freq(b *testing.B) {
|
|
||||||
l, err := NewARC(8192)
|
|
||||||
if err != nil {
|
|
||||||
b.Fatalf("err: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
trace := make([]int64, b.N*2)
|
|
||||||
for i := 0; i < b.N*2; i++ {
|
|
||||||
if i%2 == 0 {
|
|
||||||
trace[i] = rand.Int63() % 16384
|
|
||||||
} else {
|
|
||||||
trace[i] = rand.Int63() % 32768
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
b.ResetTimer()
|
|
||||||
|
|
||||||
for i := 0; i < b.N; i++ {
|
|
||||||
l.Add(trace[i], trace[i])
|
|
||||||
}
|
|
||||||
var hit, miss int
|
|
||||||
for i := 0; i < b.N; i++ {
|
|
||||||
_, ok := l.Get(trace[i])
|
|
||||||
if ok {
|
|
||||||
hit++
|
|
||||||
} else {
|
|
||||||
miss++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
b.Logf("hit: %d miss: %d ratio: %f", hit, miss, float64(hit)/float64(miss))
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestARC_RandomOps(t *testing.T) {
|
|
||||||
size := 128
|
|
||||||
l, err := NewARC(128)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("err: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
n := 200000
|
|
||||||
for i := 0; i < n; i++ {
|
|
||||||
key := rand.Int63() % 512
|
|
||||||
r := rand.Int63()
|
|
||||||
switch r % 3 {
|
|
||||||
case 0:
|
|
||||||
l.Add(key, key)
|
|
||||||
case 1:
|
|
||||||
l.Get(key)
|
|
||||||
case 2:
|
|
||||||
l.Remove(key)
|
|
||||||
}
|
|
||||||
|
|
||||||
if l.t1.Len()+l.t2.Len() > size {
|
|
||||||
t.Fatalf("bad: t1: %d t2: %d b1: %d b2: %d p: %d",
|
|
||||||
l.t1.Len(), l.t2.Len(), l.b1.Len(), l.b2.Len(), l.p)
|
|
||||||
}
|
|
||||||
if l.b1.Len()+l.b2.Len() > size {
|
|
||||||
t.Fatalf("bad: t1: %d t2: %d b1: %d b2: %d p: %d",
|
|
||||||
l.t1.Len(), l.t2.Len(), l.b1.Len(), l.b2.Len(), l.p)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestARC_Get_RecentToFrequent(t *testing.T) {
|
|
||||||
l, err := NewARC(128)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("err: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Touch all the entries, should be in t1
|
|
||||||
for i := 0; i < 128; i++ {
|
|
||||||
l.Add(i, i)
|
|
||||||
}
|
|
||||||
if n := l.t1.Len(); n != 128 {
|
|
||||||
t.Fatalf("bad: %d", n)
|
|
||||||
}
|
|
||||||
if n := l.t2.Len(); n != 0 {
|
|
||||||
t.Fatalf("bad: %d", n)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get should upgrade to t2
|
|
||||||
for i := 0; i < 128; i++ {
|
|
||||||
_, ok := l.Get(i)
|
|
||||||
if !ok {
|
|
||||||
t.Fatalf("missing: %d", i)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if n := l.t1.Len(); n != 0 {
|
|
||||||
t.Fatalf("bad: %d", n)
|
|
||||||
}
|
|
||||||
if n := l.t2.Len(); n != 128 {
|
|
||||||
t.Fatalf("bad: %d", n)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get be from t2
|
|
||||||
for i := 0; i < 128; i++ {
|
|
||||||
_, ok := l.Get(i)
|
|
||||||
if !ok {
|
|
||||||
t.Fatalf("missing: %d", i)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if n := l.t1.Len(); n != 0 {
|
|
||||||
t.Fatalf("bad: %d", n)
|
|
||||||
}
|
|
||||||
if n := l.t2.Len(); n != 128 {
|
|
||||||
t.Fatalf("bad: %d", n)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestARC_Add_RecentToFrequent(t *testing.T) {
|
|
||||||
l, err := NewARC(128)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("err: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add initially to t1
|
|
||||||
l.Add(1, 1)
|
|
||||||
if n := l.t1.Len(); n != 1 {
|
|
||||||
t.Fatalf("bad: %d", n)
|
|
||||||
}
|
|
||||||
if n := l.t2.Len(); n != 0 {
|
|
||||||
t.Fatalf("bad: %d", n)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add should upgrade to t2
|
|
||||||
l.Add(1, 1)
|
|
||||||
if n := l.t1.Len(); n != 0 {
|
|
||||||
t.Fatalf("bad: %d", n)
|
|
||||||
}
|
|
||||||
if n := l.t2.Len(); n != 1 {
|
|
||||||
t.Fatalf("bad: %d", n)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add should remain in t2
|
|
||||||
l.Add(1, 1)
|
|
||||||
if n := l.t1.Len(); n != 0 {
|
|
||||||
t.Fatalf("bad: %d", n)
|
|
||||||
}
|
|
||||||
if n := l.t2.Len(); n != 1 {
|
|
||||||
t.Fatalf("bad: %d", n)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestARC_Adaptive(t *testing.T) {
|
|
||||||
l, err := NewARC(4)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("err: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Fill t1
|
|
||||||
for i := 0; i < 4; i++ {
|
|
||||||
l.Add(i, i)
|
|
||||||
}
|
|
||||||
if n := l.t1.Len(); n != 4 {
|
|
||||||
t.Fatalf("bad: %d", n)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Move to t2
|
|
||||||
l.Get(0)
|
|
||||||
l.Get(1)
|
|
||||||
if n := l.t2.Len(); n != 2 {
|
|
||||||
t.Fatalf("bad: %d", n)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Evict from t1
|
|
||||||
l.Add(4, 4)
|
|
||||||
if n := l.b1.Len(); n != 1 {
|
|
||||||
t.Fatalf("bad: %d", n)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Current state
|
|
||||||
// t1 : (MRU) [4, 3] (LRU)
|
|
||||||
// t2 : (MRU) [1, 0] (LRU)
|
|
||||||
// b1 : (MRU) [2] (LRU)
|
|
||||||
// b2 : (MRU) [] (LRU)
|
|
||||||
|
|
||||||
// Add 2, should cause hit on b1
|
|
||||||
l.Add(2, 2)
|
|
||||||
if n := l.b1.Len(); n != 1 {
|
|
||||||
t.Fatalf("bad: %d", n)
|
|
||||||
}
|
|
||||||
if l.p != 1 {
|
|
||||||
t.Fatalf("bad: %d", l.p)
|
|
||||||
}
|
|
||||||
if n := l.t2.Len(); n != 3 {
|
|
||||||
t.Fatalf("bad: %d", n)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Current state
|
|
||||||
// t1 : (MRU) [4] (LRU)
|
|
||||||
// t2 : (MRU) [2, 1, 0] (LRU)
|
|
||||||
// b1 : (MRU) [3] (LRU)
|
|
||||||
// b2 : (MRU) [] (LRU)
|
|
||||||
|
|
||||||
// Add 4, should migrate to t2
|
|
||||||
l.Add(4, 4)
|
|
||||||
if n := l.t1.Len(); n != 0 {
|
|
||||||
t.Fatalf("bad: %d", n)
|
|
||||||
}
|
|
||||||
if n := l.t2.Len(); n != 4 {
|
|
||||||
t.Fatalf("bad: %d", n)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Current state
|
|
||||||
// t1 : (MRU) [] (LRU)
|
|
||||||
// t2 : (MRU) [4, 2, 1, 0] (LRU)
|
|
||||||
// b1 : (MRU) [3] (LRU)
|
|
||||||
// b2 : (MRU) [] (LRU)
|
|
||||||
|
|
||||||
// Add 4, should evict to b2
|
|
||||||
l.Add(5, 5)
|
|
||||||
if n := l.t1.Len(); n != 1 {
|
|
||||||
t.Fatalf("bad: %d", n)
|
|
||||||
}
|
|
||||||
if n := l.t2.Len(); n != 3 {
|
|
||||||
t.Fatalf("bad: %d", n)
|
|
||||||
}
|
|
||||||
if n := l.b2.Len(); n != 1 {
|
|
||||||
t.Fatalf("bad: %d", n)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Current state
|
|
||||||
// t1 : (MRU) [5] (LRU)
|
|
||||||
// t2 : (MRU) [4, 2, 1] (LRU)
|
|
||||||
// b1 : (MRU) [3] (LRU)
|
|
||||||
// b2 : (MRU) [0] (LRU)
|
|
||||||
|
|
||||||
// Add 0, should decrease p
|
|
||||||
l.Add(0, 0)
|
|
||||||
if n := l.t1.Len(); n != 0 {
|
|
||||||
t.Fatalf("bad: %d", n)
|
|
||||||
}
|
|
||||||
if n := l.t2.Len(); n != 4 {
|
|
||||||
t.Fatalf("bad: %d", n)
|
|
||||||
}
|
|
||||||
if n := l.b1.Len(); n != 2 {
|
|
||||||
t.Fatalf("bad: %d", n)
|
|
||||||
}
|
|
||||||
if n := l.b2.Len(); n != 0 {
|
|
||||||
t.Fatalf("bad: %d", n)
|
|
||||||
}
|
|
||||||
if l.p != 0 {
|
|
||||||
t.Fatalf("bad: %d", l.p)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Current state
|
|
||||||
// t1 : (MRU) [] (LRU)
|
|
||||||
// t2 : (MRU) [0, 4, 2, 1] (LRU)
|
|
||||||
// b1 : (MRU) [5, 3] (LRU)
|
|
||||||
// b2 : (MRU) [0] (LRU)
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestARC(t *testing.T) {
|
|
||||||
l, err := NewARC(128)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("err: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
for i := 0; i < 256; i++ {
|
|
||||||
l.Add(i, i)
|
|
||||||
}
|
|
||||||
if l.Len() != 128 {
|
|
||||||
t.Fatalf("bad len: %v", l.Len())
|
|
||||||
}
|
|
||||||
|
|
||||||
for i, k := range l.Keys() {
|
|
||||||
if v, ok := l.Get(k); !ok || v != k || v != i+128 {
|
|
||||||
t.Fatalf("bad key: %v", k)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for i := 0; i < 128; i++ {
|
|
||||||
_, ok := l.Get(i)
|
|
||||||
if ok {
|
|
||||||
t.Fatalf("should be evicted")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for i := 128; i < 256; i++ {
|
|
||||||
_, ok := l.Get(i)
|
|
||||||
if !ok {
|
|
||||||
t.Fatalf("should not be evicted")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for i := 128; i < 192; i++ {
|
|
||||||
l.Remove(i)
|
|
||||||
_, ok := l.Get(i)
|
|
||||||
if ok {
|
|
||||||
t.Fatalf("should be deleted")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
l.Purge()
|
|
||||||
if l.Len() != 0 {
|
|
||||||
t.Fatalf("bad len: %v", l.Len())
|
|
||||||
}
|
|
||||||
if _, ok := l.Get(200); ok {
|
|
||||||
t.Fatalf("should contain nothing")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Test that Contains doesn't update recent-ness
|
|
||||||
func TestARC_Contains(t *testing.T) {
|
|
||||||
l, err := NewARC(2)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("err: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
l.Add(1, 1)
|
|
||||||
l.Add(2, 2)
|
|
||||||
if !l.Contains(1) {
|
|
||||||
t.Errorf("1 should be contained")
|
|
||||||
}
|
|
||||||
|
|
||||||
l.Add(3, 3)
|
|
||||||
if l.Contains(1) {
|
|
||||||
t.Errorf("Contains should not have updated recent-ness of 1")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Test that Peek doesn't update recent-ness
|
|
||||||
func TestARC_Peek(t *testing.T) {
|
|
||||||
l, err := NewARC(2)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("err: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
l.Add(1, 1)
|
|
||||||
l.Add(2, 2)
|
|
||||||
if v, ok := l.Peek(1); !ok || v != 1 {
|
|
||||||
t.Errorf("1 should be set to 1: %v, %v", v, ok)
|
|
||||||
}
|
|
||||||
|
|
||||||
l.Add(3, 3)
|
|
||||||
if l.Contains(1) {
|
|
||||||
t.Errorf("should not have updated recent-ness of 1")
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,114 +0,0 @@
|
||||||
// This package provides a simple LRU cache. It is based on the
|
|
||||||
// LRU implementation in groupcache:
|
|
||||||
// https://github.com/golang/groupcache/tree/master/lru
|
|
||||||
package lru
|
|
||||||
|
|
||||||
import (
|
|
||||||
"sync"
|
|
||||||
|
|
||||||
"github.com/hashicorp/golang-lru/simplelru"
|
|
||||||
)
|
|
||||||
|
|
||||||
// Cache is a thread-safe fixed size LRU cache.
|
|
||||||
type Cache struct {
|
|
||||||
lru *simplelru.LRU
|
|
||||||
lock sync.RWMutex
|
|
||||||
}
|
|
||||||
|
|
||||||
// New creates an LRU of the given size
|
|
||||||
func New(size int) (*Cache, error) {
|
|
||||||
return NewWithEvict(size, nil)
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewWithEvict constructs a fixed size cache with the given eviction
|
|
||||||
// callback.
|
|
||||||
func NewWithEvict(size int, onEvicted func(key interface{}, value interface{})) (*Cache, error) {
|
|
||||||
lru, err := simplelru.NewLRU(size, simplelru.EvictCallback(onEvicted))
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
c := &Cache{
|
|
||||||
lru: lru,
|
|
||||||
}
|
|
||||||
return c, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Purge is used to completely clear the cache
|
|
||||||
func (c *Cache) Purge() {
|
|
||||||
c.lock.Lock()
|
|
||||||
c.lru.Purge()
|
|
||||||
c.lock.Unlock()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add adds a value to the cache. Returns true if an eviction occurred.
|
|
||||||
func (c *Cache) Add(key, value interface{}) bool {
|
|
||||||
c.lock.Lock()
|
|
||||||
defer c.lock.Unlock()
|
|
||||||
return c.lru.Add(key, value)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get looks up a key's value from the cache.
|
|
||||||
func (c *Cache) Get(key interface{}) (interface{}, bool) {
|
|
||||||
c.lock.Lock()
|
|
||||||
defer c.lock.Unlock()
|
|
||||||
return c.lru.Get(key)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check if a key is in the cache, without updating the recent-ness
|
|
||||||
// or deleting it for being stale.
|
|
||||||
func (c *Cache) Contains(key interface{}) bool {
|
|
||||||
c.lock.RLock()
|
|
||||||
defer c.lock.RUnlock()
|
|
||||||
return c.lru.Contains(key)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Returns the key value (or undefined if not found) without updating
|
|
||||||
// the "recently used"-ness of the key.
|
|
||||||
func (c *Cache) Peek(key interface{}) (interface{}, bool) {
|
|
||||||
c.lock.RLock()
|
|
||||||
defer c.lock.RUnlock()
|
|
||||||
return c.lru.Peek(key)
|
|
||||||
}
|
|
||||||
|
|
||||||
// ContainsOrAdd checks if a key is in the cache without updating the
|
|
||||||
// recent-ness or deleting it for being stale, and if not, adds the value.
|
|
||||||
// Returns whether found and whether an eviction occurred.
|
|
||||||
func (c *Cache) ContainsOrAdd(key, value interface{}) (ok, evict bool) {
|
|
||||||
c.lock.Lock()
|
|
||||||
defer c.lock.Unlock()
|
|
||||||
|
|
||||||
if c.lru.Contains(key) {
|
|
||||||
return true, false
|
|
||||||
} else {
|
|
||||||
evict := c.lru.Add(key, value)
|
|
||||||
return false, evict
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Remove removes the provided key from the cache.
|
|
||||||
func (c *Cache) Remove(key interface{}) {
|
|
||||||
c.lock.Lock()
|
|
||||||
c.lru.Remove(key)
|
|
||||||
c.lock.Unlock()
|
|
||||||
}
|
|
||||||
|
|
||||||
// RemoveOldest removes the oldest item from the cache.
|
|
||||||
func (c *Cache) RemoveOldest() {
|
|
||||||
c.lock.Lock()
|
|
||||||
c.lru.RemoveOldest()
|
|
||||||
c.lock.Unlock()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Keys returns a slice of the keys in the cache, from oldest to newest.
|
|
||||||
func (c *Cache) Keys() []interface{} {
|
|
||||||
c.lock.RLock()
|
|
||||||
defer c.lock.RUnlock()
|
|
||||||
return c.lru.Keys()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Len returns the number of items in the cache.
|
|
||||||
func (c *Cache) Len() int {
|
|
||||||
c.lock.RLock()
|
|
||||||
defer c.lock.RUnlock()
|
|
||||||
return c.lru.Len()
|
|
||||||
}
|
|
|
@ -1,221 +0,0 @@
|
||||||
package lru
|
|
||||||
|
|
||||||
import (
|
|
||||||
"math/rand"
|
|
||||||
"testing"
|
|
||||||
)
|
|
||||||
|
|
||||||
func BenchmarkLRU_Rand(b *testing.B) {
|
|
||||||
l, err := New(8192)
|
|
||||||
if err != nil {
|
|
||||||
b.Fatalf("err: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
trace := make([]int64, b.N*2)
|
|
||||||
for i := 0; i < b.N*2; i++ {
|
|
||||||
trace[i] = rand.Int63() % 32768
|
|
||||||
}
|
|
||||||
|
|
||||||
b.ResetTimer()
|
|
||||||
|
|
||||||
var hit, miss int
|
|
||||||
for i := 0; i < 2*b.N; i++ {
|
|
||||||
if i%2 == 0 {
|
|
||||||
l.Add(trace[i], trace[i])
|
|
||||||
} else {
|
|
||||||
_, ok := l.Get(trace[i])
|
|
||||||
if ok {
|
|
||||||
hit++
|
|
||||||
} else {
|
|
||||||
miss++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
b.Logf("hit: %d miss: %d ratio: %f", hit, miss, float64(hit)/float64(miss))
|
|
||||||
}
|
|
||||||
|
|
||||||
func BenchmarkLRU_Freq(b *testing.B) {
|
|
||||||
l, err := New(8192)
|
|
||||||
if err != nil {
|
|
||||||
b.Fatalf("err: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
trace := make([]int64, b.N*2)
|
|
||||||
for i := 0; i < b.N*2; i++ {
|
|
||||||
if i%2 == 0 {
|
|
||||||
trace[i] = rand.Int63() % 16384
|
|
||||||
} else {
|
|
||||||
trace[i] = rand.Int63() % 32768
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
b.ResetTimer()
|
|
||||||
|
|
||||||
for i := 0; i < b.N; i++ {
|
|
||||||
l.Add(trace[i], trace[i])
|
|
||||||
}
|
|
||||||
var hit, miss int
|
|
||||||
for i := 0; i < b.N; i++ {
|
|
||||||
_, ok := l.Get(trace[i])
|
|
||||||
if ok {
|
|
||||||
hit++
|
|
||||||
} else {
|
|
||||||
miss++
|
|
||||||
}
|
|
||||||
}
|
|
||||||
b.Logf("hit: %d miss: %d ratio: %f", hit, miss, float64(hit)/float64(miss))
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestLRU(t *testing.T) {
|
|
||||||
evictCounter := 0
|
|
||||||
onEvicted := func(k interface{}, v interface{}) {
|
|
||||||
if k != v {
|
|
||||||
t.Fatalf("Evict values not equal (%v!=%v)", k, v)
|
|
||||||
}
|
|
||||||
evictCounter += 1
|
|
||||||
}
|
|
||||||
l, err := NewWithEvict(128, onEvicted)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("err: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
for i := 0; i < 256; i++ {
|
|
||||||
l.Add(i, i)
|
|
||||||
}
|
|
||||||
if l.Len() != 128 {
|
|
||||||
t.Fatalf("bad len: %v", l.Len())
|
|
||||||
}
|
|
||||||
|
|
||||||
if evictCounter != 128 {
|
|
||||||
t.Fatalf("bad evict count: %v", evictCounter)
|
|
||||||
}
|
|
||||||
|
|
||||||
for i, k := range l.Keys() {
|
|
||||||
if v, ok := l.Get(k); !ok || v != k || v != i+128 {
|
|
||||||
t.Fatalf("bad key: %v", k)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for i := 0; i < 128; i++ {
|
|
||||||
_, ok := l.Get(i)
|
|
||||||
if ok {
|
|
||||||
t.Fatalf("should be evicted")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for i := 128; i < 256; i++ {
|
|
||||||
_, ok := l.Get(i)
|
|
||||||
if !ok {
|
|
||||||
t.Fatalf("should not be evicted")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for i := 128; i < 192; i++ {
|
|
||||||
l.Remove(i)
|
|
||||||
_, ok := l.Get(i)
|
|
||||||
if ok {
|
|
||||||
t.Fatalf("should be deleted")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
l.Get(192) // expect 192 to be last key in l.Keys()
|
|
||||||
|
|
||||||
for i, k := range l.Keys() {
|
|
||||||
if (i < 63 && k != i+193) || (i == 63 && k != 192) {
|
|
||||||
t.Fatalf("out of order key: %v", k)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
l.Purge()
|
|
||||||
if l.Len() != 0 {
|
|
||||||
t.Fatalf("bad len: %v", l.Len())
|
|
||||||
}
|
|
||||||
if _, ok := l.Get(200); ok {
|
|
||||||
t.Fatalf("should contain nothing")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// test that Add returns true/false if an eviction occurred
|
|
||||||
func TestLRUAdd(t *testing.T) {
|
|
||||||
evictCounter := 0
|
|
||||||
onEvicted := func(k interface{}, v interface{}) {
|
|
||||||
evictCounter += 1
|
|
||||||
}
|
|
||||||
|
|
||||||
l, err := NewWithEvict(1, onEvicted)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("err: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if l.Add(1, 1) == true || evictCounter != 0 {
|
|
||||||
t.Errorf("should not have an eviction")
|
|
||||||
}
|
|
||||||
if l.Add(2, 2) == false || evictCounter != 1 {
|
|
||||||
t.Errorf("should have an eviction")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// test that Contains doesn't update recent-ness
|
|
||||||
func TestLRUContains(t *testing.T) {
|
|
||||||
l, err := New(2)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("err: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
l.Add(1, 1)
|
|
||||||
l.Add(2, 2)
|
|
||||||
if !l.Contains(1) {
|
|
||||||
t.Errorf("1 should be contained")
|
|
||||||
}
|
|
||||||
|
|
||||||
l.Add(3, 3)
|
|
||||||
if l.Contains(1) {
|
|
||||||
t.Errorf("Contains should not have updated recent-ness of 1")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// test that Contains doesn't update recent-ness
|
|
||||||
func TestLRUContainsOrAdd(t *testing.T) {
|
|
||||||
l, err := New(2)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("err: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
l.Add(1, 1)
|
|
||||||
l.Add(2, 2)
|
|
||||||
contains, evict := l.ContainsOrAdd(1, 1)
|
|
||||||
if !contains {
|
|
||||||
t.Errorf("1 should be contained")
|
|
||||||
}
|
|
||||||
if evict {
|
|
||||||
t.Errorf("nothing should be evicted here")
|
|
||||||
}
|
|
||||||
|
|
||||||
l.Add(3, 3)
|
|
||||||
contains, evict = l.ContainsOrAdd(1, 1)
|
|
||||||
if contains {
|
|
||||||
t.Errorf("1 should not have been contained")
|
|
||||||
}
|
|
||||||
if !evict {
|
|
||||||
t.Errorf("an eviction should have occurred")
|
|
||||||
}
|
|
||||||
if !l.Contains(1) {
|
|
||||||
t.Errorf("now 1 should be contained")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// test that Peek doesn't update recent-ness
|
|
||||||
func TestLRUPeek(t *testing.T) {
|
|
||||||
l, err := New(2)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("err: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
l.Add(1, 1)
|
|
||||||
l.Add(2, 2)
|
|
||||||
if v, ok := l.Peek(1); !ok || v != 1 {
|
|
||||||
t.Errorf("1 should be set to 1: %v, %v", v, ok)
|
|
||||||
}
|
|
||||||
|
|
||||||
l.Add(3, 3)
|
|
||||||
if l.Contains(1) {
|
|
||||||
t.Errorf("should not have updated recent-ness of 1")
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,160 +0,0 @@
|
||||||
package simplelru
|
|
||||||
|
|
||||||
import (
|
|
||||||
"container/list"
|
|
||||||
"errors"
|
|
||||||
)
|
|
||||||
|
|
||||||
// EvictCallback is used to get a callback when a cache entry is evicted
|
|
||||||
type EvictCallback func(key interface{}, value interface{})
|
|
||||||
|
|
||||||
// LRU implements a non-thread safe fixed size LRU cache
|
|
||||||
type LRU struct {
|
|
||||||
size int
|
|
||||||
evictList *list.List
|
|
||||||
items map[interface{}]*list.Element
|
|
||||||
onEvict EvictCallback
|
|
||||||
}
|
|
||||||
|
|
||||||
// entry is used to hold a value in the evictList
|
|
||||||
type entry struct {
|
|
||||||
key interface{}
|
|
||||||
value interface{}
|
|
||||||
}
|
|
||||||
|
|
||||||
// NewLRU constructs an LRU of the given size
|
|
||||||
func NewLRU(size int, onEvict EvictCallback) (*LRU, error) {
|
|
||||||
if size <= 0 {
|
|
||||||
return nil, errors.New("Must provide a positive size")
|
|
||||||
}
|
|
||||||
c := &LRU{
|
|
||||||
size: size,
|
|
||||||
evictList: list.New(),
|
|
||||||
items: make(map[interface{}]*list.Element),
|
|
||||||
onEvict: onEvict,
|
|
||||||
}
|
|
||||||
return c, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Purge is used to completely clear the cache
|
|
||||||
func (c *LRU) Purge() {
|
|
||||||
for k, v := range c.items {
|
|
||||||
if c.onEvict != nil {
|
|
||||||
c.onEvict(k, v.Value.(*entry).value)
|
|
||||||
}
|
|
||||||
delete(c.items, k)
|
|
||||||
}
|
|
||||||
c.evictList.Init()
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add adds a value to the cache. Returns true if an eviction occurred.
|
|
||||||
func (c *LRU) Add(key, value interface{}) bool {
|
|
||||||
// Check for existing item
|
|
||||||
if ent, ok := c.items[key]; ok {
|
|
||||||
c.evictList.MoveToFront(ent)
|
|
||||||
ent.Value.(*entry).value = value
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
// Add new item
|
|
||||||
ent := &entry{key, value}
|
|
||||||
entry := c.evictList.PushFront(ent)
|
|
||||||
c.items[key] = entry
|
|
||||||
|
|
||||||
evict := c.evictList.Len() > c.size
|
|
||||||
// Verify size not exceeded
|
|
||||||
if evict {
|
|
||||||
c.removeOldest()
|
|
||||||
}
|
|
||||||
return evict
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get looks up a key's value from the cache.
|
|
||||||
func (c *LRU) Get(key interface{}) (value interface{}, ok bool) {
|
|
||||||
if ent, ok := c.items[key]; ok {
|
|
||||||
c.evictList.MoveToFront(ent)
|
|
||||||
return ent.Value.(*entry).value, true
|
|
||||||
}
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check if a key is in the cache, without updating the recent-ness
|
|
||||||
// or deleting it for being stale.
|
|
||||||
func (c *LRU) Contains(key interface{}) (ok bool) {
|
|
||||||
_, ok = c.items[key]
|
|
||||||
return ok
|
|
||||||
}
|
|
||||||
|
|
||||||
// Returns the key value (or undefined if not found) without updating
|
|
||||||
// the "recently used"-ness of the key.
|
|
||||||
func (c *LRU) Peek(key interface{}) (value interface{}, ok bool) {
|
|
||||||
if ent, ok := c.items[key]; ok {
|
|
||||||
return ent.Value.(*entry).value, true
|
|
||||||
}
|
|
||||||
return nil, ok
|
|
||||||
}
|
|
||||||
|
|
||||||
// Remove removes the provided key from the cache, returning if the
|
|
||||||
// key was contained.
|
|
||||||
func (c *LRU) Remove(key interface{}) bool {
|
|
||||||
if ent, ok := c.items[key]; ok {
|
|
||||||
c.removeElement(ent)
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
|
|
||||||
// RemoveOldest removes the oldest item from the cache.
|
|
||||||
func (c *LRU) RemoveOldest() (interface{}, interface{}, bool) {
|
|
||||||
ent := c.evictList.Back()
|
|
||||||
if ent != nil {
|
|
||||||
c.removeElement(ent)
|
|
||||||
kv := ent.Value.(*entry)
|
|
||||||
return kv.key, kv.value, true
|
|
||||||
}
|
|
||||||
return nil, nil, false
|
|
||||||
}
|
|
||||||
|
|
||||||
// GetOldest returns the oldest entry
|
|
||||||
func (c *LRU) GetOldest() (interface{}, interface{}, bool) {
|
|
||||||
ent := c.evictList.Back()
|
|
||||||
if ent != nil {
|
|
||||||
kv := ent.Value.(*entry)
|
|
||||||
return kv.key, kv.value, true
|
|
||||||
}
|
|
||||||
return nil, nil, false
|
|
||||||
}
|
|
||||||
|
|
||||||
// Keys returns a slice of the keys in the cache, from oldest to newest.
|
|
||||||
func (c *LRU) Keys() []interface{} {
|
|
||||||
keys := make([]interface{}, len(c.items))
|
|
||||||
i := 0
|
|
||||||
for ent := c.evictList.Back(); ent != nil; ent = ent.Prev() {
|
|
||||||
keys[i] = ent.Value.(*entry).key
|
|
||||||
i++
|
|
||||||
}
|
|
||||||
return keys
|
|
||||||
}
|
|
||||||
|
|
||||||
// Len returns the number of items in the cache.
|
|
||||||
func (c *LRU) Len() int {
|
|
||||||
return c.evictList.Len()
|
|
||||||
}
|
|
||||||
|
|
||||||
// removeOldest removes the oldest item from the cache.
|
|
||||||
func (c *LRU) removeOldest() {
|
|
||||||
ent := c.evictList.Back()
|
|
||||||
if ent != nil {
|
|
||||||
c.removeElement(ent)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// removeElement is used to remove a given list element from the cache
|
|
||||||
func (c *LRU) removeElement(e *list.Element) {
|
|
||||||
c.evictList.Remove(e)
|
|
||||||
kv := e.Value.(*entry)
|
|
||||||
delete(c.items, kv.key)
|
|
||||||
if c.onEvict != nil {
|
|
||||||
c.onEvict(kv.key, kv.value)
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,167 +0,0 @@
|
||||||
package simplelru
|
|
||||||
|
|
||||||
import "testing"
|
|
||||||
|
|
||||||
func TestLRU(t *testing.T) {
|
|
||||||
evictCounter := 0
|
|
||||||
onEvicted := func(k interface{}, v interface{}) {
|
|
||||||
if k != v {
|
|
||||||
t.Fatalf("Evict values not equal (%v!=%v)", k, v)
|
|
||||||
}
|
|
||||||
evictCounter += 1
|
|
||||||
}
|
|
||||||
l, err := NewLRU(128, onEvicted)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("err: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
for i := 0; i < 256; i++ {
|
|
||||||
l.Add(i, i)
|
|
||||||
}
|
|
||||||
if l.Len() != 128 {
|
|
||||||
t.Fatalf("bad len: %v", l.Len())
|
|
||||||
}
|
|
||||||
|
|
||||||
if evictCounter != 128 {
|
|
||||||
t.Fatalf("bad evict count: %v", evictCounter)
|
|
||||||
}
|
|
||||||
|
|
||||||
for i, k := range l.Keys() {
|
|
||||||
if v, ok := l.Get(k); !ok || v != k || v != i+128 {
|
|
||||||
t.Fatalf("bad key: %v", k)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for i := 0; i < 128; i++ {
|
|
||||||
_, ok := l.Get(i)
|
|
||||||
if ok {
|
|
||||||
t.Fatalf("should be evicted")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for i := 128; i < 256; i++ {
|
|
||||||
_, ok := l.Get(i)
|
|
||||||
if !ok {
|
|
||||||
t.Fatalf("should not be evicted")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for i := 128; i < 192; i++ {
|
|
||||||
ok := l.Remove(i)
|
|
||||||
if !ok {
|
|
||||||
t.Fatalf("should be contained")
|
|
||||||
}
|
|
||||||
ok = l.Remove(i)
|
|
||||||
if ok {
|
|
||||||
t.Fatalf("should not be contained")
|
|
||||||
}
|
|
||||||
_, ok = l.Get(i)
|
|
||||||
if ok {
|
|
||||||
t.Fatalf("should be deleted")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
l.Get(192) // expect 192 to be last key in l.Keys()
|
|
||||||
|
|
||||||
for i, k := range l.Keys() {
|
|
||||||
if (i < 63 && k != i+193) || (i == 63 && k != 192) {
|
|
||||||
t.Fatalf("out of order key: %v", k)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
l.Purge()
|
|
||||||
if l.Len() != 0 {
|
|
||||||
t.Fatalf("bad len: %v", l.Len())
|
|
||||||
}
|
|
||||||
if _, ok := l.Get(200); ok {
|
|
||||||
t.Fatalf("should contain nothing")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestLRU_GetOldest_RemoveOldest(t *testing.T) {
|
|
||||||
l, err := NewLRU(128, nil)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("err: %v", err)
|
|
||||||
}
|
|
||||||
for i := 0; i < 256; i++ {
|
|
||||||
l.Add(i, i)
|
|
||||||
}
|
|
||||||
k, _, ok := l.GetOldest()
|
|
||||||
if !ok {
|
|
||||||
t.Fatalf("missing")
|
|
||||||
}
|
|
||||||
if k.(int) != 128 {
|
|
||||||
t.Fatalf("bad: %v", k)
|
|
||||||
}
|
|
||||||
|
|
||||||
k, _, ok = l.RemoveOldest()
|
|
||||||
if !ok {
|
|
||||||
t.Fatalf("missing")
|
|
||||||
}
|
|
||||||
if k.(int) != 128 {
|
|
||||||
t.Fatalf("bad: %v", k)
|
|
||||||
}
|
|
||||||
|
|
||||||
k, _, ok = l.RemoveOldest()
|
|
||||||
if !ok {
|
|
||||||
t.Fatalf("missing")
|
|
||||||
}
|
|
||||||
if k.(int) != 129 {
|
|
||||||
t.Fatalf("bad: %v", k)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Test that Add returns true/false if an eviction occurred
|
|
||||||
func TestLRU_Add(t *testing.T) {
|
|
||||||
evictCounter := 0
|
|
||||||
onEvicted := func(k interface{}, v interface{}) {
|
|
||||||
evictCounter += 1
|
|
||||||
}
|
|
||||||
|
|
||||||
l, err := NewLRU(1, onEvicted)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("err: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
if l.Add(1, 1) == true || evictCounter != 0 {
|
|
||||||
t.Errorf("should not have an eviction")
|
|
||||||
}
|
|
||||||
if l.Add(2, 2) == false || evictCounter != 1 {
|
|
||||||
t.Errorf("should have an eviction")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Test that Contains doesn't update recent-ness
|
|
||||||
func TestLRU_Contains(t *testing.T) {
|
|
||||||
l, err := NewLRU(2, nil)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("err: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
l.Add(1, 1)
|
|
||||||
l.Add(2, 2)
|
|
||||||
if !l.Contains(1) {
|
|
||||||
t.Errorf("1 should be contained")
|
|
||||||
}
|
|
||||||
|
|
||||||
l.Add(3, 3)
|
|
||||||
if l.Contains(1) {
|
|
||||||
t.Errorf("Contains should not have updated recent-ness of 1")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Test that Peek doesn't update recent-ness
|
|
||||||
func TestLRU_Peek(t *testing.T) {
|
|
||||||
l, err := NewLRU(2, nil)
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("err: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
l.Add(1, 1)
|
|
||||||
l.Add(2, 2)
|
|
||||||
if v, ok := l.Peek(1); !ok || v != 1 {
|
|
||||||
t.Errorf("1 should be set to 1: %v, %v", v, ok)
|
|
||||||
}
|
|
||||||
|
|
||||||
l.Add(3, 3)
|
|
||||||
if l.Contains(1) {
|
|
||||||
t.Errorf("should not have updated recent-ness of 1")
|
|
||||||
}
|
|
||||||
}
|
|
67
video.go
67
video.go
|
@ -1,67 +0,0 @@
|
||||||
package thumbnail
|
|
||||||
|
|
||||||
import (
|
|
||||||
"bytes"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"os/exec"
|
|
||||||
)
|
|
||||||
|
|
||||||
func (this *Thumbnailer) RenderScaledFfmpeg(absPath string) ([]byte, error) {
|
|
||||||
|
|
||||||
scaleCmd := fmt.Sprintf(
|
|
||||||
`thumbnail,scale='if(gt(a,%d/%d),%d,-1)':'if(gt(a,%d/%d),-1,%d)'`,
|
|
||||||
this.height, this.width, this.height,
|
|
||||||
this.height, this.width, this.width,
|
|
||||||
)
|
|
||||||
|
|
||||||
// BUG(): always produces ASPECT_SVELTE shape - need to re-pad for ASPECT_PADDED
|
|
||||||
|
|
||||||
var vcodec string
|
|
||||||
switch this.ofmt {
|
|
||||||
case OUTPUT_JPG:
|
|
||||||
vcodec = "mjpeg" // yes really
|
|
||||||
case OUTPUT_PNG_CRUSH:
|
|
||||||
vcodec = "png"
|
|
||||||
default:
|
|
||||||
return nil, ErrInvalidOption
|
|
||||||
}
|
|
||||||
|
|
||||||
cmd := exec.Command(
|
|
||||||
"ffmpeg",
|
|
||||||
"-loglevel", "0",
|
|
||||||
"-timelimit", "10", // seconds
|
|
||||||
"-an",
|
|
||||||
"-i", absPath,
|
|
||||||
"-vf", scaleCmd,
|
|
||||||
"-frames:v", "1",
|
|
||||||
"-f", "image2pipe",
|
|
||||||
"-c:v", vcodec,
|
|
||||||
"-",
|
|
||||||
)
|
|
||||||
|
|
||||||
// -ss 00:00:30
|
|
||||||
|
|
||||||
stdout, err := cmd.StdoutPipe()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
err = cmd.Start()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
out := bytes.Buffer{}
|
|
||||||
_, err = io.Copy(&out, stdout)
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
err = cmd.Wait()
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return out.Bytes(), nil
|
|
||||||
}
|
|
Loading…
Reference in New Issue